OILS / ysh / grammar.pgen2 View on Github | oils.pub

545 lines, 185 significant
1# Grammar for YSH.
2# Adapted from the Python 3.7 expression grammar, with several changes!
3#
4# TODO:
5# - List comprehensions
6# - There's also chaining => and maybe implicit vectorization ==>
7# - But list comprehensions are more familiar, and they are concise
8# - Generator expressions?
9# - Do we need lambdas?
10
11# Note: trailing commas are allowed:
12# {k: mydict,}
13# [mylist,]
14# mytuple,
15# f(args,)
16# func f(params,)
17#
18# Kinds used:
19# VSub, Left, Right, Expr, Op, Arith, Char, Eof, Unknown
20
21# YSH patch: removed @=
22augassign: (
23 '+=' | '-=' | '*=' | '/=' |
24 '**=' | '//=' | '%=' |
25 '&=' | '|=' | '^=' | '<<=' | '>>='
26)
27
28test: or_test ['if' or_test 'else' test] | lambdef
29
30# Lambdas follow the same rules as Python:
31#
32# |x| 1, 2 == (|x| 1), 2
33# |x| x if True else 42 == |x| (x if True else 42)
34#
35# Python also had a test_nocond production like this: We don't need it because
36# we can't have multiple ifs.
37# [x for x in range(3) if lambda x: x if 1]
38#
39# The zero arg syntax like || 1 annoys me -- but this also works:
40# func() { return 1 }
41#
42# We used name_type_list rather than param_group because a default value like
43# x|y (bitwise or) conflicts with the | delimiter!
44#
45# TODO: consider this syntax:
46# fn (x) x # expression
47# fn (x) ^( echo hi ) # statement
48
49lambdef: '|' [name_type_list] '|' test
50
51or_test: and_test ('or' and_test)*
52and_test: not_test ('and' not_test)*
53not_test: 'not' not_test | comparison
54comparison: range_expr (comp_op range_expr)*
55
56# Unlike slice, beginning and end are required
57range_expr: (
58 expr ['..<' expr] |
59 expr ['..=' expr]
60)
61
62# YSH patch: remove legacy <>, add === and more
63comp_op: (
64 '<'|'>'|'==='|'>='|'<='|'!=='|'in'|'not' 'in'|'is'|'is' 'not'|
65 '~' | '!~' | '~~' | '!~~' | '~=='
66)
67
68# For lists and dicts. Note: In Python this was star_expr *foo
69splat_expr: '...' expr
70
71expr: xor_expr ('|' xor_expr)*
72xor_expr: and_expr ('^' and_expr)*
73and_expr: shift_expr ('&' shift_expr)*
74shift_expr: arith_expr (('<<'|'>>') arith_expr)*
75# YSH: add concatenation ++ with same precedence as +
76arith_expr: term (('+'|'-'|'++') term)*
77# YSH: removed '@' matrix mul
78term: factor (('*'|'/'|'//'|'%') factor)*
79factor: ('+'|'-'|'~') factor | power
80# YSH: removed Python 3 'await'
81power: atom trailer* ['**' factor]
82
83# Note: I think splat_expr is not for list comprehensions, it's only for
84# literals like [42, *x] in Python, or [42, ...x] in YSH. This is new in
85# Python 3.
86# I think splat_expr expressed awkwardly because of pgen limitations.
87testlist_comp: (test|splat_expr) ( comp_for | (',' (test|splat_expr))* [','] )
88
89atom: (
90 '(' [testlist_comp] ')' # empty tuple/list, or parenthesized expression
91 | '[' [testlist_comp] ']' # empty list or list comprehension
92 | '^[' testlist ']' # expression literal
93 # note: ^[x for x in y] is invalid
94 # but ^[[x for x in y]] is a list comprehension
95
96 # Note: newlines are significant inside {}, unlike inside () and []
97 | '{' [Op_Newline] [dict] '}'
98 | '&' Expr_Name place_trailer*
99
100 # NOTE: These atoms are are allowed in typed array literals
101 | Expr_Name | Expr_Null | Expr_True | Expr_False
102
103 # Allow suffixes on floats and decimals
104 # e.g. 100 M is a function M which multiplies by 1_000_000
105 # e.g. 100 Mi is a function Mi which multiplies by 1024 * 1024
106 | Expr_Float [Expr_Name]
107 | Expr_DecInt [Expr_Name]
108
109 | Expr_BinInt | Expr_OctInt | Expr_HexInt
110
111 | Char_OneChar # char literal \n \\ etc.
112 | Char_YHex
113 | Char_UBraced # char literal \u{3bc}
114
115 | dq_string | sq_string
116 # Expr_Symbol could be %mykey
117
118 | eggex
119
120 # $foo is disallowed, but $? is allowed. Should be "$foo" to indicate a
121 # string, or ${foo:-}
122 | simple_var_sub
123 | sh_command_sub | braced_var_sub
124 | sh_array_literal
125 | old_sh_array_literal
126)
127
128place_trailer: (
129 '[' subscriptlist ']'
130 | '.' Expr_Name
131)
132
133# var f = f(x)
134trailer: (
135 '(' [arglist] ')'
136 | '[' subscriptlist ']'
137
138 # Is a {} trailing useful for anything? It's not in Python or JS
139
140 | '.' Expr_Name
141 | '->' Expr_Name
142 | '=>' Expr_Name
143)
144
145# YSH patch: this is 'expr' instead of 'test'
146# - 1:(3<4) doesn't make sense.
147# - TODO: could we revert this? I think it might have been because we wanted
148# first class slices like var x = 1:n, but we have ranges var x = 1 .. n instead.
149# - There was also the colon conflict for :symbol
150
151subscriptlist: subscript (',' subscript)* [',']
152
153# TODO: Add => as low precedence operator, for Func[Str, Int => Str]
154subscript: expr | [expr] ':' [expr]
155
156# TODO: => should be even lower precedence here too
157testlist: test (',' test)* [',']
158
159# Dict syntax resembles JavaScript
160# https://stackoverflow.com/questions/38948306/what-is-javascript-shorthand-property
161#
162# Examples:
163# {age: 20} is like {'age': 20}
164#
165# x = 'age'
166# d = %{[x]: 20} # Evaluate x as a variable
167# d = %{["foo$x"]: 20} # Another expression
168# d = %{[x, y]: 20} # Tuple key
169# d = %{key1, key1: 123}
170# Notes:
171# - Value is optional when the key is a name, because it can be taken from the
172# environment.
173# - We don't have:
174# - dict comprehensions. Maybe wait until LR parsing?
175# - Splatting with **
176
177dict_pair: (
178 Expr_Name [':' test]
179 | '[' testlist ']' ':' test
180 | sq_string ':' test
181 | dq_string ':' test
182)
183
184comma_newline: ',' [Op_Newline] | Op_Newline
185
186dict: dict_pair (comma_newline dict_pair)* [comma_newline]
187
188# This how Python implemented dict comprehensions. We can probably do the
189# same.
190#
191# dictorsetmaker: ( ((test ':' test | '**' expr)
192# (comp_for | (',' (test ':' test | '**' expr))* [','])) |
193# ((test | splat_expr)
194# (comp_for | (',' (test | splat_expr))* [','])) )
195
196# The reason that keywords are test nodes instead of NAME is that using NAME
197# results in an ambiguity. ast.c makes sure it's a NAME.
198# "test '=' test" is really "keyword '=' test", but we have no such token.
199# These need to be in a single rule to avoid grammar that is ambiguous
200# to our LL(1) parser. Even though 'test' includes '*expr' in splat_expr,
201# we explicitly match '*' here, too, to give it proper precedence.
202# Illegal combinations and orderings are blocked in ast.c:
203# multiple (test comp_for) arguments are blocked; keyword unpackings
204# that precede iterable unpackings are blocked; etc.
205
206argument: (
207 test [comp_for]
208 # named arg
209 | test '=' test
210 # splat. The ... goes before, not after, to be consistent with Python, JS,
211 # and the prefix @ operator.
212 | '...' test
213)
214
215# The grammar at call sites is less restrictive than at declaration sites.
216# ... can appear anywhere. Keyword args can appear anywhere too.
217arg_group: argument (',' argument)* [',']
218arglist: (
219 [arg_group]
220 [';' [arg_group]]
221)
222arglist3: (
223 [arg_group]
224 [';' [arg_group]]
225 [';' [argument]] # procs have an extra block argument
226)
227
228
229# YSH patch: test_nocond -> or_test. I believe this was trying to prevent the
230# "double if" ambiguity here:
231# #
232# [x for x in range(3) if lambda x: x if 1]
233#
234# but YSH doesn't supported "nested loops", so we don't have this problem.
235comp_for: 'for' name_type_list 'in' or_test ['if' or_test]
236
237
238#
239# Expressions that are New in YSH
240#
241
242# Notes:
243# - Most of these occur in 'atom' above
244# - You can write $mystr but not mystr. It has to be (mystr)
245array_item: (
246 Expr_Null | Expr_True | Expr_False
247 | Expr_Float | Expr_DecInt | Expr_BinInt | Expr_OctInt | Expr_HexInt
248 | dq_string | sq_string
249 | sh_command_sub | braced_var_sub | simple_var_sub
250 | '(' test ')'
251)
252sh_array_literal: ':|' Expr_CastedDummy Op_Pipe
253
254# TODO: remove old array
255old_sh_array_literal: '%(' Expr_CastedDummy Right_ShArrayLiteral
256sh_command_sub: ( '$(' | '@(' | '^(' ) Expr_CastedDummy Eof_RParen
257
258# " $" """ $""" ^"
259dq_string: (
260 Left_DoubleQuote | Left_DollarDoubleQuote |
261 Left_TDoubleQuote | Left_DollarTDoubleQuote |
262 Left_CaretDoubleQuote
263 ) Expr_CastedDummy Right_DoubleQuote
264
265# ' ''' r' r'''
266# $' for "refactoring" property
267# u' u''' b' b'''
268sq_string: (
269 Left_SingleQuote | Left_TSingleQuote
270 | Left_RSingleQuote | Left_RTSingleQuote
271 | Left_DollarSingleQuote
272 | Left_USingleQuote | Left_UTSingleQuote
273 | Left_BSingleQuote | Left_BTSingleQuote
274) Expr_CastedDummy Right_SingleQuote
275
276braced_var_sub: '${' Expr_CastedDummy Right_DollarBrace
277
278simple_var_sub: (
279 # This is everything in Kind.VSub except VSub_Name, which is braced: ${foo}
280 #
281 # Note: we could allow $foo and $0, but disallow the rest in favor of ${@}
282 # and ${-}? Meh it's too inconsistent.
283 VSub_DollarName | VSub_Number
284 | VSub_Bang | VSub_At | VSub_Pound | VSub_Dollar | VSub_Star | VSub_Hyphen
285 | VSub_QMark
286 # NOTE: $? should be STATUS because it's an integer.
287)
288
289#
290# Assignment / Type Variables
291#
292# Several differences vs. Python:
293#
294# - no yield expression on RHS
295# - no star expressions on either side (Python 3) *x, y = 2, *b
296# - no multiple assignments like: var x = y = 3
297# - type annotation syntax is more restrictive # a: (1+2) = 3 is OK in python
298# - We're validating the lvalue here, instead of doing it in the "transformer".
299# We have the 'var' prefix which helps.
300
301# name_type use cases:
302# var x Int, y Int = 3, 5
303# / <capture d+ as date: int> /
304#
305# for x Int, y Int
306# [x for x Int, y Int in ...]
307#
308# func(x Int, y Int) - this is separate
309
310# Optional colon because we want both
311
312# var x: Int = 42 # colon looks nicer
313# proc p (; x Int, y Int; z Int) { echo hi } # colon gets in the way of ;
314
315name_type: Expr_Name [':'] [type_expr]
316name_type_list: name_type (',' name_type)*
317
318type_expr: Expr_Name [ '[' type_expr (',' type_expr)* ']' ]
319
320# NOTE: Eof_RParen and Eof_Backtick aren't allowed because we don't want 'var'
321# in command subs.
322end_stmt: '}' | ';' | Op_Newline | Eof_Real
323
324# TODO: allow -> to denote aliasing/mutation
325ysh_var_decl: name_type_list ['=' testlist] end_stmt
326
327# Note: this is more precise way of writing ysh_mutation, but it's ambiguous :(
328# ysh_mutation: lhs augassign testlist end_stmt
329# | lhs_list '=' testlist end_stmt
330
331# Note: for YSH (not Tea), we could accept [':'] expr for setvar :out = 'foo'
332lhs_list: expr (',' expr)*
333
334# TODO: allow -> to denote aliasing/mutation
335ysh_mutation: lhs_list (augassign | '=') testlist end_stmt
336
337# proc arg lists, like:
338# json write (x, indent=1)
339# cd /tmp ( ; ; ^(echo hi))
340#
341# What about:
342# myproc /tmp [ ; ; ^(echo hi)] - I guess this doesn't make sense?
343ysh_eager_arglist: '(' [arglist3] ')'
344ysh_lazy_arglist: '[' [arglist] ']'
345
346#
347# Other Entry Points
348#
349
350# if (x > 0) etc.
351ysh_expr: '(' testlist ')'
352
353# = 42 + a[i]
354# call f(x)
355command_expr: testlist end_stmt
356
357# $[d->key] etc.
358ysh_expr_sub: testlist ']'
359
360# Signatures for proc and func.
361
362# Note: 'proc name-with-hyphens' is allowed, so we can't parse the name in
363# expression mode.
364ysh_proc: (
365 [ '('
366 [ param_group ] # word params, with defaults
367 [ ';' [ param_group ] ] # positional typed params, with defaults
368 [ ';' [ param_group ] ] # named params, with defaults
369 [ ';' [ param_group ] ] # optional block param, with no type or default
370
371 # This causes a pgen2 error? It doesn't know which branch to take
372 # So we have the extra {block} syntax
373 #[ ';' Expr_Name ] # optional block param, with no type or default
374 ')'
375 ]
376 '{' # opening { for pgen2
377)
378
379ysh_func: (
380 Expr_Name '(' [param_group] [';' param_group] ')' ['=>' type_expr] '{'
381)
382
383param: Expr_Name [type_expr] ['=' expr]
384
385# This is an awkward way of writing that '...' has to come last.
386param_group: (
387 (param ',')*
388 [ (param | '...' Expr_Name) [','] ]
389)
390
391#
392# Regex Sublanguage
393#
394
395char_literal: Char_OneChar | Char_Hex | Char_UBraced
396
397# we allow a-z A-Z 0-9 as ranges, but otherwise they have to be quoted
398# The parser enforces that they are single strings
399range_char: Expr_Name | Expr_DecInt | sq_string | char_literal
400
401# digit or a-z
402# We have to do further validation of ranges later.
403class_literal_term: (
404 # NOTE: range_char has sq_string
405 range_char ['-' range_char ]
406 # splice a literal set of characters
407 | '@' Expr_Name
408 | '!' Expr_Name
409 # Reserved for [[.collating sequences.]] (Unicode)
410 | '.' Expr_Name
411 # Reserved for [[=character equivalents=]] (Unicode)
412 | '=' Expr_Name
413 # TODO: Do these char classes actually work in bash/awk/egrep/sed/etc.?
414
415)
416class_literal: '[' class_literal_term+ ']'
417
418# NOTE: Here is an example of where you can put ^ in the middle of a pattern in
419# Python, and it matters!
420# >>> r = re.compile('.f[a-z]*', re.DOTALL|re.MULTILINE)
421# >>> r.findall('z\nfoo\nbeef\nfood\n')
422# ['\nfoo', 'ef', '\nfood']
423# >>> r = re.compile('.^f[a-z]*', re.DOTALL|re.MULTILINE)
424# r.findall('z\nfoo\nbeef\nfood\n')
425# ['\nfoo', '\nfood']
426
427re_atom: (
428 char_literal
429 # builtin regex like 'digit' or a regex reference like 'D'
430 | Expr_Name
431 # %begin or %end
432 | Expr_Symbol
433 | class_literal
434 # !digit or ![a-f]. Note ! %boundary could be \B in Python, but ERE
435 # doesn't have anything like that
436 | '!' (Expr_Name | class_literal)
437
438 # syntactic space for Perl-style backtracking
439 # !!REF 1 !!REF name
440 # !!AHEAD(d+) !!BEHIND(d+) !!NOT_AHEAD(d+) !!NOT_BEHIND(d+)
441 #
442 # Note: !! conflicts with history
443 | '!' '!' Expr_Name (Expr_Name | Expr_DecInt | '(' regex ')')
444
445 # Splice another expression
446 | '@' Expr_Name
447 # any %start %end are preferred
448 | '.' | '^' | '$'
449 # In a language-independent spec, backslashes are disallowed within 'sq'.
450 # Write it with char literals outside strings: 'foo' \\ 'bar' \n
451 #
452 # No double-quoted strings because you can write "x = $x" with 'x = ' @x
453 | sq_string
454
455 # grouping (non-capturing in Perl; capturing in ERE although < > is preferred)
456 | '(' regex ')'
457
458 # Capturing group, with optional name and conversion function
459 # <capture d+ as date>
460 # <capture d+ as date: int>
461 # <capture d+ : int>
462 | '<' 'capture' regex ['as' Expr_Name] [':' Expr_Name] '>'
463
464 # Might want this obscure conditional construct. Can't use C-style ternary
465 # because '?' is a regex operator.
466 #| '{' regex 'if' regex 'else' regex '}'
467
468 # Others:
469 # PCRE has (?R ) for recursion? That could be !RECURSE()
470 # Note: .NET has && in character classes, making it a recursive language
471)
472
473# e.g. a{3} a{3,4} a{3,} a{,4} but not a{,}
474repeat_range: (
475 Expr_DecInt [',']
476 | ',' Expr_DecInt
477 | Expr_DecInt ',' Expr_DecInt
478)
479
480repeat_op: (
481 '+' | '*' | '?'
482 # In PCRE, ?? *? +? {}? is lazy/nongreedy and ?+ *+ ++ {}+ is "possessive"
483 # We use N and P modifiers within {}.
484 # a{L +} a{P ?} a{P 3,4} a{P ,4}
485 | '{' [Expr_Name] ('+' | '*' | '?' | repeat_range) '}'
486)
487
488re_alt: (re_atom [repeat_op])+
489
490regex: [re_alt] (('|'|'or') re_alt)*
491
492# e.g. /digit+ ; multiline !ignorecase/
493#
494# This can express translation preferences:
495#
496# / d+ ; ; ERE / is '[[:digit:]]+'
497# / d+ ; ; PCRE / is '\d+'
498# / d+ ; ignorecase ; python / is '(?i)\d+'
499
500# Python has the syntax
501# (?i:myre) to set a flag
502# (?-i:myre) to remove a flag
503#
504# They can apply to portions of the expression, which we don't have here.
505re_flag: ['!'] Expr_Name
506eggex: '/' regex [';' re_flag* [';' Expr_Name] ] '/'
507
508# Patterns are the start of a case arm. Ie,
509#
510# case (foo) {
511# (40 + 2) | (0) { echo number }
512# ^^^^^^^^^^^^^^-- This is pattern
513# }
514#
515# Due to limitations created from pgen2/cmd_parser interactions, we also parse
516# the leading '{' token of the case arm body in pgen2. We do this to help pgen2
517# figure out when to transfer control back to the cmd_parser. For more details
518# see #oil-dev > Dev Friction / Smells.
519#
520# case (foo) {
521# (40 + 2) | (0) { echo number }
522# ^-- End of pattern/beginning of case arm body
523# }
524
525ysh_case_pat: (
526 '(' (pat_else | pat_exprs)
527 | eggex
528) [Op_Newline] '{'
529
530pat_else: 'else' ')'
531pat_exprs: expr ')' [Op_Newline] ('|' [Op_Newline] '(' expr ')' [Op_Newline])*
532
533
534# Syntax reserved for PCRE/Python, but that's not in ERE:
535#
536# non-greedy a{N *}
537# non-capturing ( digit+ )
538# backtracking !!REF 1 !!AHEAD(d+)
539#
540# Legacy syntax:
541#
542# ^ and $ instead of %start and %end
543# < and > instead of %start_word and %end_word
544# . instead of dot
545# | instead of 'or'