| 1 | #!/usr/bin/env bash
|
| 2 | #
|
| 3 | # Proof of concept for pgen2 and YSH syntax.
|
| 4 | #
|
| 5 | # Usage:
|
| 6 | # ./pgen2-test.sh <function name>
|
| 7 |
|
| 8 | set -o nounset
|
| 9 | set -o pipefail
|
| 10 | set -o errexit
|
| 11 |
|
| 12 | banner() {
|
| 13 | echo
|
| 14 | echo "----- $@ -----"
|
| 15 | echo
|
| 16 | }
|
| 17 |
|
| 18 | grammar-gen() {
|
| 19 | PYTHONPATH=. ysh/grammar_gen.py "$@"
|
| 20 | }
|
| 21 |
|
| 22 | # Build the grammar and parse code. Outside of the Oils binary.
|
| 23 | parse() {
|
| 24 | grammar-gen parse "$@"
|
| 25 | }
|
| 26 |
|
| 27 | parse-exprs() {
|
| 28 | readonly -a exprs=(
|
| 29 | '1+2'
|
| 30 | '1 + 2 * 3'
|
| 31 | 'x | ~y'
|
| 32 | '1 << x'
|
| 33 | 'a not in b'
|
| 34 | 'a is not b'
|
| 35 | '[x for x in a]'
|
| 36 | '[1, 2]'
|
| 37 | '{myset, a}'
|
| 38 | '{mydict: a, key: b}'
|
| 39 | '{x: dictcomp for x in b}'
|
| 40 | 'a[1,2]'
|
| 41 | 'a[i:i+1]'
|
| 42 | )
|
| 43 | for expr in "${exprs[@]}"; do
|
| 44 | parse pgen2/oil.grammar eval_input "$expr"
|
| 45 | done
|
| 46 | }
|
| 47 |
|
| 48 | parse-arglists() {
|
| 49 | readonly -a arglists=(
|
| 50 | 'a'
|
| 51 | 'a,b'
|
| 52 | 'a,b=1'
|
| 53 | # Hm this parses, although isn't not valid
|
| 54 | 'a=1,b'
|
| 55 | 'a, *b, **kwargs'
|
| 56 |
|
| 57 | # Hm how is this valid?
|
| 58 |
|
| 59 | # Comment:
|
| 60 | # "The reason that keywords are test nodes instead of NAME is that using
|
| 61 | # NAME results in an ambiguity. ast.c makes sure it's a NAME."
|
| 62 | #
|
| 63 | # Hm is the parsing model powerful enough?
|
| 64 | # TODO: change it to NAME and figure out what happens.
|
| 65 | #
|
| 66 | # Python 3.6's grammar has more comments!
|
| 67 |
|
| 68 | # "test '=' test" is really "keyword '=' test", but we have no such token.
|
| 69 | # These need to be in a single rule to avoid grammar that is ambiguous
|
| 70 | # to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
|
| 71 | # we explicitly match '*' here, too, to give it proper precedence.
|
| 72 | # Illegal combinations and orderings are blocked in ast.c:
|
| 73 | # multiple (test comp_for) arguments are blocked; keyword unpackings
|
| 74 | # that precede iterable unpackings are blocked; etc.
|
| 75 |
|
| 76 | 'a+1'
|
| 77 | )
|
| 78 |
|
| 79 | for expr in "${arglists[@]}"; do
|
| 80 | parse pgen2/oil.grammar arglist_input "$expr"
|
| 81 | done
|
| 82 | }
|
| 83 |
|
| 84 | # NOTE: Unused small demo.
|
| 85 | parse-types() {
|
| 86 | readonly -a types=(
|
| 87 | 'int'
|
| 88 | 'str'
|
| 89 | 'List<str>'
|
| 90 | 'Tuple<str, int, int>'
|
| 91 | 'Dict<str, int>'
|
| 92 | # aha! Tokenizer issue
|
| 93 | #'Dict<str, Tuple<int, int>>'
|
| 94 |
|
| 95 | # Must be like this! That's funny.
|
| 96 | 'Dict<str, Tuple<int, int> >'
|
| 97 | )
|
| 98 | for expr in "${types[@]}"; do
|
| 99 | parse pgen2/oil.grammar type_input "$expr"
|
| 100 | done
|
| 101 | }
|
| 102 |
|
| 103 | readonly OIL_GRAMMAR='ysh/grammar.pgen2'
|
| 104 |
|
| 105 | calc-test() {
|
| 106 | local -a exprs=(
|
| 107 | 'a + 2'
|
| 108 | '1 + 2*3/4' # operator precedence and left assoc
|
| 109 |
|
| 110 | # Tuple
|
| 111 | 'x+1, y+1'
|
| 112 | #'(x+1, y+1)' # TODO: atom
|
| 113 |
|
| 114 | # Associative
|
| 115 | '-1+2+3'
|
| 116 | '4*5*6'
|
| 117 | 'i % n'
|
| 118 | 'i % n / 2'
|
| 119 |
|
| 120 | # Uses string tokens
|
| 121 | #'"abc" + "def"'
|
| 122 |
|
| 123 | '2 ^ 3 ^ 4' # right assoc
|
| 124 | 'f(1)'
|
| 125 | 'f(1, 2, 3)'
|
| 126 |
|
| 127 | 'f(a[i], 2, 3)'
|
| 128 | 'f(a[i, j], 2, 3)'
|
| 129 |
|
| 130 | 'f(x)^3'
|
| 131 | 'f(x)[i]^3'
|
| 132 |
|
| 133 | #'x < 3 and y <= 4'
|
| 134 |
|
| 135 | # bad token
|
| 136 | #'a * 3&4'
|
| 137 | )
|
| 138 |
|
| 139 | for e in "${exprs[@]}"; do
|
| 140 | echo "$e"
|
| 141 | parse $OIL_GRAMMAR eval_input "$e"
|
| 142 | done
|
| 143 | }
|
| 144 |
|
| 145 | oil-productions() {
|
| 146 | parse $OIL_GRAMMAR oil_var 'a = 1;'
|
| 147 | parse $OIL_GRAMMAR oil_var 'a Int = 2;'
|
| 148 |
|
| 149 | # Invalid because += now allowed
|
| 150 | #parse $OIL_GRAMMAR oil_var 'a += 1;'
|
| 151 |
|
| 152 | parse $OIL_GRAMMAR oil_setvar 'x = 3;'
|
| 153 | parse $OIL_GRAMMAR oil_setvar 'x += 4;'
|
| 154 |
|
| 155 | # Invalid because type expression isn't allowed (it could conflict)
|
| 156 | #parse $OIL_GRAMMAR oil_setvar 'x Int += 4;'
|
| 157 | }
|
| 158 |
|
| 159 | mode-test() {
|
| 160 | # Test all the mode transitions
|
| 161 | local -a exprs=(
|
| 162 | # Expr -> Array
|
| 163 | # TODO: how is OilOuter different than Array
|
| 164 | '@[]'
|
| 165 | 'x + @[a b] + y'
|
| 166 |
|
| 167 | # Expr -> Command
|
| 168 | # Hm empty could be illegal?
|
| 169 | '$[]'
|
| 170 | 'x + $[hi there] + y'
|
| 171 |
|
| 172 | # Expr -> Expr
|
| 173 | '$(x)'
|
| 174 | # NOTE: operator precedence is respected here!
|
| 175 | 'x + $(f(y) - 3) * 4'
|
| 176 | # Expr -> Expr even though we saw )
|
| 177 | #'$(f(x, y) + (1 * 3))'
|
| 178 |
|
| 179 | # Expr -> OilVS
|
| 180 | #'${}' # syntax error
|
| 181 | '${x}'
|
| 182 | # This will work when we add | to grammar
|
| 183 | #'x + ${p|html} + y'
|
| 184 |
|
| 185 | # Expr -> Regex
|
| 186 | #'$/ /'
|
| 187 | 'x + $/ mypat / + y' # syntactically valid, semantically invalid
|
| 188 |
|
| 189 | # Expr -> OilDQ
|
| 190 | '"hello \$"'
|
| 191 | 'x + "hello \$" + y'
|
| 192 | # TODO: Also do every other kind of string:
|
| 193 | # r'raw' r"raw $sub" ''' """ r''' r"""
|
| 194 |
|
| 195 | # Regex -> CharClass
|
| 196 | #'$/ any* "." [a-z A-Z _] [a-z A-Z _ 0-9]+ /'
|
| 197 | '$/ "." [a-z A-Z _] [a-z A-Z _ 0-9] /'
|
| 198 | '$/ a [b] c /'
|
| 199 |
|
| 200 | # Array -> CharClass
|
| 201 | '@[one two *.[c h] *.[NOT c h] ]'
|
| 202 |
|
| 203 | # Expr -> Array -> CharClass
|
| 204 | 'left + @[one two *.[c h] ] + right'
|
| 205 | # Array brace sub. Not PARSED yet, but no lexer mode change AFAICT
|
| 206 | #'@[ -{one,two}- *.[c h] ]'
|
| 207 |
|
| 208 | ## OilDQ -> Expr
|
| 209 | '"var expr $(2 + 3)"'
|
| 210 |
|
| 211 | ## OilDQ -> Command
|
| 212 | '"command $[echo hi]"'
|
| 213 |
|
| 214 | # OilDQ -> OilVS -- % is not an operator
|
| 215 | #'"quoted ${x %02d}"'
|
| 216 | '"quoted ${x}"'
|
| 217 |
|
| 218 | #)
|
| 219 | #local -a exprs=(
|
| 220 |
|
| 221 | )
|
| 222 |
|
| 223 | for e in "${exprs[@]}"; do
|
| 224 | echo "$e"
|
| 225 | parse $OIL_GRAMMAR eval_input "$e"
|
| 226 | done
|
| 227 |
|
| 228 | # Command stuff. TODO: we don't have a parser for this!
|
| 229 | # Maybe add 'echo' do everything?
|
| 230 | exprs+=(
|
| 231 | #'x = $[echo one; echo *.[c h] ]'
|
| 232 |
|
| 233 | # Command -> Expr (PROBLEM: requires lookahead to =)
|
| 234 | 'x = a + b'
|
| 235 | 'var x = a + b'
|
| 236 | 'setvar x = a + b'
|
| 237 |
|
| 238 | # Command -> Expr
|
| 239 | 'echo $(a + b)'
|
| 240 | 'echo ${x|html}'
|
| 241 |
|
| 242 | # Command -> Expr
|
| 243 | 'echo $stringfunc(x, y)'
|
| 244 | 'echo @arrayfunc(x, y)'
|
| 245 |
|
| 246 | # The signature must be parsed expression mode if it have
|
| 247 | # defaults.
|
| 248 | 'func foo(x Int, y Int = 42 + 1) Int {
|
| 249 | echo $x $y
|
| 250 | }
|
| 251 | '
|
| 252 | # I guess [] is parsed in expression mode too. It's a very simple grammar.
|
| 253 | # It only accepts strings. Maybe there is a special "BLOCK" var you can
|
| 254 | # evaluate.
|
| 255 | 'proc copy [src dest="default $value"] {
|
| 256 | echo $src $dest
|
| 257 | }
|
| 258 | '
|
| 259 |
|
| 260 | 'if (x > 1) { echo hi }'
|
| 261 |
|
| 262 | 'while (x > 0) {
|
| 263 | set x -= 1
|
| 264 | }
|
| 265 | '
|
| 266 | 'for (x in y) { # "var" is implied; error if x is already defined?
|
| 267 | echo $y
|
| 268 | }
|
| 269 | '
|
| 270 | 'for (i = 0; i < 10; ++i) {
|
| 271 | echo $i
|
| 272 | }
|
| 273 | '
|
| 274 | 'switch (i+1) {
|
| 275 | case 1:
|
| 276 | echo "one"
|
| 277 | case 2:
|
| 278 | echo "two"
|
| 279 | }
|
| 280 | '
|
| 281 | 'match (x) {
|
| 282 | 1 { echo "one" }
|
| 283 | 2 { echo "two" }
|
| 284 | }
|
| 285 | '
|
| 286 |
|
| 287 | # Command -> OilVS -- % is not an operator
|
| 288 | 'echo ${x %02d}'
|
| 289 |
|
| 290 | # Command -> CharClass is DISALLOWED. Must go through array?
|
| 291 | # @() could be synonym for array expression.
|
| 292 | # Although if you could come up with a custom syntax error for this: it
|
| 293 | # might be OK.
|
| 294 | # a[x] = 1
|
| 295 | #'echo *.[c h]'
|
| 296 | #
|
| 297 | # I think you could restrict the first words
|
| 298 | )
|
| 299 |
|
| 300 | # I don't think these are essential.
|
| 301 | local -a deferred=(
|
| 302 | # Expr -> Command (PROBLEM: mode is grammatical; needs state machine)
|
| 303 | 'x = func(x, y={}) {
|
| 304 | echo hi
|
| 305 | }
|
| 306 | '
|
| 307 |
|
| 308 | # Expr -> Command (PROBLEM: ditto)
|
| 309 | # This one is even harder, because technically the expression on the left
|
| 310 | # could have {}? Or we can ban that in patterns?
|
| 311 | 'x = match(x) {
|
| 312 | 1 { echo one }
|
| 313 | 2 { echo two }
|
| 314 | }
|
| 315 | '
|
| 316 |
|
| 317 | # stays in Expr for comparison
|
| 318 | 'x = match(x) {
|
| 319 | 1 => "one"
|
| 320 | 2 => "two"
|
| 321 | }
|
| 322 | '
|
| 323 | )
|
| 324 | }
|
| 325 |
|
| 326 | enum-test() {
|
| 327 | readonly -a enums=(
|
| 328 | # second alternative
|
| 329 | 'for 3 a'
|
| 330 | 'for 3 { a, b }'
|
| 331 | 'for 3 a { a, b }'
|
| 332 | #'for'
|
| 333 | #'a'
|
| 334 | )
|
| 335 | for expr in "${enums[@]}"; do
|
| 336 | parse pgen2/enum.grammar eval_input "$expr"
|
| 337 | done
|
| 338 | }
|
| 339 |
|
| 340 | all() {
|
| 341 | banner 'exprs'
|
| 342 | parse-exprs
|
| 343 |
|
| 344 | #banner 'arglists'
|
| 345 | #parse-arglists
|
| 346 |
|
| 347 | banner 'calc'
|
| 348 | calc-test
|
| 349 |
|
| 350 | banner 'mode-test'
|
| 351 | mode-test
|
| 352 |
|
| 353 | banner 'oil-productions'
|
| 354 | oil-productions
|
| 355 |
|
| 356 | # enum-test doesn't work?
|
| 357 | }
|
| 358 |
|
| 359 | # Hm Python 3 has type syntax! But we may not use it.
|
| 360 | # And it has async/await.
|
| 361 | # And walrus operator :=.
|
| 362 | # @ matrix multiplication operator.
|
| 363 |
|
| 364 | diff-grammars() {
|
| 365 | wc -l ~/src/languages/Python-*/Grammar/Grammar
|
| 366 |
|
| 367 | cdiff ~/src/languages/Python-{2.7.15,3.7.3}/Grammar/Grammar
|
| 368 | }
|
| 369 |
|
| 370 | stdlib-test() {
|
| 371 | pgen2 stdlib-test
|
| 372 | }
|
| 373 |
|
| 374 | "$@"
|