| 1 | #!/usr/bin/env python2
|
| 2 | # Copyright 2016 Andy Chu. All rights reserved.
|
| 3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 | # you may not use this file except in compliance with the License.
|
| 5 | # You may obtain a copy of the License at
|
| 6 | #
|
| 7 | # http://www.apache.org/licenses/LICENSE-2.0
|
| 8 | """
|
| 9 | id_kind_def.py - Id and Kind definitions, stored in Token
|
| 10 |
|
| 11 | NOTE: If this file changes, rebuild it with build/py.sh all
|
| 12 | """
|
| 13 | from __future__ import print_function
|
| 14 |
|
| 15 | from _devbuild.gen.types_asdl import (bool_arg_type_e, bool_arg_type_t)
|
| 16 | #from mycpp.mylib import log
|
| 17 |
|
| 18 | from typing import List, Tuple, Dict, Optional, TYPE_CHECKING
|
| 19 | if TYPE_CHECKING: # avoid circular build deps
|
| 20 | from _devbuild.gen.id_kind_asdl import Id_t, Kind_t
|
| 21 |
|
| 22 |
|
| 23 | class IdSpec(object):
|
| 24 | """Identifiers that form the "spine" of the shell program
|
| 25 | representation."""
|
| 26 |
|
| 27 | def __init__(self, kind_lookup, bool_ops):
|
| 28 | # type: (Dict[int, int], Dict[int, bool_arg_type_t]) -> None
|
| 29 | self.id_str2int = {} # type: Dict[str, int]
|
| 30 | self.kind_str2int = {} # type: Dict[str, int]
|
| 31 |
|
| 32 | self.kind_lookup = kind_lookup # Id int -> Kind int
|
| 33 | self.kind_name_list = [] # type: List[str]
|
| 34 | self.kind_sizes = [] # type: List[int] # optional stats
|
| 35 |
|
| 36 | self.lexer_pairs = {} # type: Dict[int, List[Tuple[bool, str, int]]]
|
| 37 | self.bool_ops = bool_ops # type: Dict[int, bool_arg_type_t]
|
| 38 |
|
| 39 | # Incremented on each method call
|
| 40 | # IMPORTANT: 1-based indices match what asdl/gen_python.py does!!!
|
| 41 | self.id_index = 1
|
| 42 | self.kind_index = 1
|
| 43 |
|
| 44 | def LexerPairs(self, kind):
|
| 45 | # type: (Kind_t) -> List[Tuple[bool, str, Id_t]]
|
| 46 | result = []
|
| 47 | for is_regex, pat, id_ in self.lexer_pairs[kind]:
|
| 48 | result.append((is_regex, pat, id_))
|
| 49 | return result
|
| 50 |
|
| 51 | def _AddId(self, id_name, kind=None):
|
| 52 | # type: (str, Optional[int]) -> int
|
| 53 | """
|
| 54 | Args:
|
| 55 | id_name: e.g. BoolBinary_Equal
|
| 56 | kind: override autoassignment. For AddBoolBinaryForBuiltin
|
| 57 | """
|
| 58 | t = self.id_index
|
| 59 |
|
| 60 | self.id_str2int[id_name] = t
|
| 61 |
|
| 62 | if kind is None:
|
| 63 | kind = self.kind_index
|
| 64 | self.kind_lookup[t] = kind
|
| 65 |
|
| 66 | self.id_index += 1 # mutate last
|
| 67 | return t # the index we used
|
| 68 |
|
| 69 | def _AddKind(self, kind_name):
|
| 70 | # type: (str) -> None
|
| 71 | self.kind_str2int[kind_name] = self.kind_index
|
| 72 | #log('%s = %d', kind_name, self.kind_index)
|
| 73 | self.kind_index += 1
|
| 74 | self.kind_name_list.append(kind_name)
|
| 75 |
|
| 76 | def AddKind(self, kind_name, tokens):
|
| 77 | # type: (str, List[str]) -> None
|
| 78 | assert isinstance(tokens, list), tokens
|
| 79 |
|
| 80 | for name in tokens:
|
| 81 | id_name = '%s_%s' % (kind_name, name)
|
| 82 | self._AddId(id_name)
|
| 83 |
|
| 84 | # Must be after adding Id
|
| 85 | self._AddKind(kind_name)
|
| 86 | self.kind_sizes.append(len(tokens)) # debug info
|
| 87 |
|
| 88 | def AddKindPairs(self, kind_name, pairs):
|
| 89 | # type: (str, List[Tuple[str, str]]) -> None
|
| 90 | assert isinstance(pairs, list), pairs
|
| 91 |
|
| 92 | lexer_pairs = []
|
| 93 | for name, char_pat in pairs:
|
| 94 | id_name = '%s_%s' % (kind_name, name)
|
| 95 | id_int = self._AddId(id_name)
|
| 96 | # After _AddId
|
| 97 | lexer_pairs.append((False, char_pat, id_int)) # Constant
|
| 98 |
|
| 99 | self.lexer_pairs[self.kind_index] = lexer_pairs
|
| 100 |
|
| 101 | # Must be after adding Id
|
| 102 | self._AddKind(kind_name)
|
| 103 | self.kind_sizes.append(len(pairs)) # debug info
|
| 104 |
|
| 105 | def AddBoolKind(
|
| 106 | self,
|
| 107 | kind_name, # type: str
|
| 108 | arg_type_pairs, # type: List[Tuple[bool_arg_type_t, List[Tuple[str, str]]]]
|
| 109 | ):
|
| 110 | # type: (...) -> None
|
| 111 | """
|
| 112 | Args:
|
| 113 | kind_name: string
|
| 114 | arg_type_pairs: dictionary of bool_arg_type_e -> []
|
| 115 | """
|
| 116 | lexer_pairs = []
|
| 117 | num_tokens = 0
|
| 118 | for arg_type, pairs in arg_type_pairs:
|
| 119 | #print(arg_type, pairs)
|
| 120 |
|
| 121 | for name, char_pat in pairs:
|
| 122 | # BoolUnary_f, BoolBinary_eq, BoolBinary_NEqual
|
| 123 | id_name = '%s_%s' % (kind_name, name)
|
| 124 | id_int = self._AddId(id_name)
|
| 125 | self.AddBoolOp(id_int, arg_type) # register type
|
| 126 | lexer_pairs.append((False, char_pat, id_int)) # constant
|
| 127 |
|
| 128 | num_tokens += len(pairs)
|
| 129 |
|
| 130 | self.lexer_pairs[self.kind_index] = lexer_pairs
|
| 131 |
|
| 132 | # Must do this after _AddId()
|
| 133 | self._AddKind(kind_name)
|
| 134 | self.kind_sizes.append(num_tokens) # debug info
|
| 135 |
|
| 136 | def AddBoolBinaryForBuiltin(self, id_name, kind):
|
| 137 | # type: (str, int) -> int
|
| 138 | """For [ = ] [ == ] and [ != ].
|
| 139 |
|
| 140 | These operators are NOT added to the lexer. The are "lexed" as
|
| 141 | word.String.
|
| 142 | """
|
| 143 | id_name = 'BoolBinary_%s' % id_name
|
| 144 | id_int = self._AddId(id_name, kind=kind)
|
| 145 | self.AddBoolOp(id_int, bool_arg_type_e.Str)
|
| 146 | return id_int
|
| 147 |
|
| 148 | def AddBoolOp(self, id_int, arg_type):
|
| 149 | # type: (int, bool_arg_type_t) -> None
|
| 150 | """Associate an ID integer with an bool_arg_type_e."""
|
| 151 | self.bool_ops[id_int] = arg_type
|
| 152 |
|
| 153 |
|
| 154 | def AddKinds(spec):
|
| 155 | # type: (IdSpec) -> None
|
| 156 |
|
| 157 | # A compound word, in arith context, boolean context, or command context.
|
| 158 | # A['foo'] A["foo"] A[$foo] A["$foo"] A[${foo}] A["${foo}"]
|
| 159 | spec.AddKind('Word', ['Compound'])
|
| 160 |
|
| 161 | # Token IDs in Kind.Arith are first to make the TDOP precedence table
|
| 162 | # small.
|
| 163 | #
|
| 164 | # NOTE: Could share Op_Pipe, Op_Amp, Op_DAmp, Op_Semi, Op_LParen, etc.
|
| 165 | # Actually all of Arith could be folded into Op, because we are using
|
| 166 | # WordParser._ReadArithWord vs. WordParser._ReadWord.
|
| 167 | spec.AddKindPairs(
|
| 168 | 'Arith',
|
| 169 | [
|
| 170 | ('Semi', ';'), # ternary for loop only
|
| 171 | ('Comma', ','), # function call and C comma operator
|
| 172 | ('Plus', '+'),
|
| 173 | ('Minus', '-'),
|
| 174 | ('Star', '*'),
|
| 175 | ('Slash', '/'),
|
| 176 | ('Percent', '%'),
|
| 177 | ('DPlus', '++'),
|
| 178 | ('DMinus', '--'),
|
| 179 | ('DStar', '**'),
|
| 180 | ('LParen', '('),
|
| 181 | ('RParen', ')'), # grouping and function call extension
|
| 182 | ('LBracket', '['),
|
| 183 | ('RBracket', ']'), # array and assoc array subscript
|
| 184 | ('RBrace', '}'), # for end of var sub
|
| 185 |
|
| 186 | # Logical Ops
|
| 187 | ('QMark', '?'),
|
| 188 | ('Colon', ':'), # Ternary Op: a < b ? 0 : 1
|
| 189 | ('LessEqual', '<='),
|
| 190 | ('Less', '<'),
|
| 191 | ('GreatEqual', '>='),
|
| 192 | ('Great', '>'),
|
| 193 | ('DEqual', '=='),
|
| 194 | ('NEqual', '!='),
|
| 195 | # note: these 3 are not in YSH Expr. (Could be used in find dialect.)
|
| 196 | ('DAmp', '&&'),
|
| 197 | ('DPipe', '||'),
|
| 198 | ('Bang', '!'),
|
| 199 |
|
| 200 | # Bitwise ops
|
| 201 | ('DGreat', '>>'),
|
| 202 | ('DLess', '<<'),
|
| 203 | # YSH: ^ is exponent
|
| 204 | ('Amp', '&'),
|
| 205 | ('Pipe', '|'),
|
| 206 | ('Caret', '^'),
|
| 207 | ('Tilde', '~'),
|
| 208 | ('Equal', '='),
|
| 209 |
|
| 210 | # Augmented Assignment for $(( ))
|
| 211 | # Must match the list in osh/arith_parse.py
|
| 212 | # YSH has **= //= like Python
|
| 213 | ('PlusEqual', '+='),
|
| 214 | ('MinusEqual', '-='),
|
| 215 | ('StarEqual', '*='),
|
| 216 | ('SlashEqual', '/='),
|
| 217 | ('PercentEqual', '%='),
|
| 218 | ('DGreatEqual', '>>='),
|
| 219 | ('DLessEqual', '<<='),
|
| 220 | ('AmpEqual', '&='),
|
| 221 | ('CaretEqual', '^='),
|
| 222 | ('PipeEqual', '|='),
|
| 223 | ])
|
| 224 |
|
| 225 | spec.AddKind('Eof', ['Real', 'RParen', 'Backtick'])
|
| 226 |
|
| 227 | spec.AddKind('Undefined', ['Tok']) # for initial state
|
| 228 |
|
| 229 | # The Unknown kind is used when we lex something, but it's invalid.
|
| 230 | # Examples:
|
| 231 | # ${^}
|
| 232 | # $'\z' Such bad codes are accepted in OSH, when no_parse_backslash is
|
| 233 | # off, so we have to lex them.
|
| 234 | # (x == y) should used === or ~==
|
| 235 | spec.AddKind('Unknown',
|
| 236 | ['Tok', 'Backslash', 'DEqual', 'DAmp', 'DPipe', 'DDot'])
|
| 237 |
|
| 238 | spec.AddKind('Eol', ['Tok']) # no more tokens on line (\0)
|
| 239 |
|
| 240 | # Ignored_Newline is for J8 lexing to count lines
|
| 241 | spec.AddKind('Ignored', ['LineCont', 'Space', 'Comment', 'Newline'])
|
| 242 |
|
| 243 | # Id.WS_Space is for lex_mode_e.ShCommand; Id.Ignored_Space is for
|
| 244 | # lex_mode_e.Arith
|
| 245 | spec.AddKind('WS', ['Space'])
|
| 246 |
|
| 247 | spec.AddKind(
|
| 248 | 'Lit',
|
| 249 | [
|
| 250 | 'Chars',
|
| 251 | 'CharsWithoutPrefix', # for stripping leading whitespace
|
| 252 | 'VarLike',
|
| 253 | 'ArrayLhsOpen',
|
| 254 | 'ArrayLhsClose',
|
| 255 | 'Splice', # @func(a, b)
|
| 256 | 'AtLBracket', # @[split(x)]
|
| 257 | 'AtLBraceDot', # @{.myproc arg1} should be builtin_sub
|
| 258 | 'Other',
|
| 259 | 'EscapedChar', # \* is escaped
|
| 260 | 'BackslashDoubleQuote', # \"
|
| 261 | 'LBracket',
|
| 262 | 'RBracket', # for assoc array literals, static globs
|
| 263 | 'Star',
|
| 264 | 'QMark',
|
| 265 | # Either brace expansion or keyword for { and }
|
| 266 | 'LBrace',
|
| 267 | 'RBrace',
|
| 268 | 'Comma',
|
| 269 | 'Equals', # For = f()
|
| 270 | 'Dollar', # detecting 'echo $'
|
| 271 | 'DRightBracket', # the ]] that matches [[, NOT a keyword
|
| 272 | 'Tilde', # tilde expansion
|
| 273 | 'Pound', # for comment or VarOp state
|
| 274 | 'TPound', # for doc comments like ###
|
| 275 | 'TDot', # for multiline commands ...
|
| 276 | 'Slash',
|
| 277 | 'Percent', # / # % for patsub, NOT unary op
|
| 278 | 'Colon', # x=foo:~:~root needs tilde expansion
|
| 279 | 'Digits', # for lex_mode_e.Arith
|
| 280 | 'At', # for ${a[@]} in lex_mode_e.Arith, and detecting @[]
|
| 281 | 'ArithVarLike', # for $((var+1)). Distinct from Lit_VarLike 'var='
|
| 282 | 'BadBackslash', # for "\z", not Id.Unknown_Backslash because it's a
|
| 283 | # syntax error in YSH, but NOT OSH
|
| 284 | 'CompDummy', # A fake Lit_* token to get partial words during
|
| 285 | # completion
|
| 286 | 'Number',
|
| 287 | 'RedirVarName', # "{myvar}", as in {myvar}>out.txt (and on its own)
|
| 288 | 'HistoryOp_PrevEntry',
|
| 289 | 'HistoryOp_WordRest',
|
| 290 | 'HistoryOp_First',
|
| 291 | 'HistoryOp_Last',
|
| 292 | 'HistoryNum',
|
| 293 | 'HistorySearch',
|
| 294 | ])
|
| 295 |
|
| 296 | # For recognizing \` and \" and \\ within backticks. There's an extra layer
|
| 297 | # of backslash quoting.
|
| 298 | spec.AddKind('Backtick', ['Right', 'Quoted', 'DoubleQuote', 'Other'])
|
| 299 |
|
| 300 | spec.AddKind('History', ['Op', 'Num', 'Search', 'Other'])
|
| 301 |
|
| 302 | spec.AddKind(
|
| 303 | 'Op',
|
| 304 | [
|
| 305 | 'Newline', # mostly equivalent to SEMI
|
| 306 | 'Amp', # &
|
| 307 | 'Pipe', # |
|
| 308 | 'PipeAmp', # |& -- bash extension for stderr
|
| 309 | 'DAmp', # &&
|
| 310 | 'DPipe', # ||
|
| 311 | 'Semi', # ;
|
| 312 | 'DSemi', # ;; for case
|
| 313 | 'SemiAmp', # ;& for case
|
| 314 | 'DSemiAmp', # ;;& for case
|
| 315 | 'LParen', # For subshell. Not Kind.Left because it's NOT a WordPart.
|
| 316 | 'RParen', # Default, will be translated to Id.Right_*
|
| 317 | 'DLeftParen',
|
| 318 | 'DRightParen',
|
| 319 |
|
| 320 | # for [[ ]] language
|
| 321 | 'Less', # <
|
| 322 | 'Great', # >
|
| 323 | 'Bang', # !
|
| 324 |
|
| 325 | # YSH [] {}
|
| 326 | 'LBracket',
|
| 327 | 'RBracket',
|
| 328 | 'LBrace',
|
| 329 | 'RBrace',
|
| 330 | ])
|
| 331 |
|
| 332 | # YSH expressions use Kind.Expr and Kind.Arith (further below)
|
| 333 | spec.AddKind(
|
| 334 | 'Expr',
|
| 335 | [
|
| 336 | 'Reserved', # <- means nothing but it's reserved now
|
| 337 | 'Symbol', # %foo
|
| 338 | 'Name',
|
| 339 | 'DecInt',
|
| 340 | 'BinInt',
|
| 341 | 'OctInt',
|
| 342 | 'HexInt',
|
| 343 | 'Float',
|
| 344 | 'Bang', # eggex !digit, ![a-z]
|
| 345 | 'Dot',
|
| 346 | 'DDotLessThan',
|
| 347 | 'DDotEqual',
|
| 348 | 'Colon', # mylist:pop()
|
| 349 | 'RArrow',
|
| 350 | 'RDArrow',
|
| 351 | 'DSlash', # integer division
|
| 352 | 'TEqual',
|
| 353 | 'NotDEqual',
|
| 354 | 'TildeDEqual', # === !== ~==
|
| 355 | 'At',
|
| 356 | 'DoubleAt', # splice operators
|
| 357 | 'Ellipsis', # for varargs
|
| 358 | 'Dollar', # legacy regex
|
| 359 | 'NotTilde', # !~
|
| 360 | 'DTilde',
|
| 361 | 'NotDTilde', # ~~ !~~
|
| 362 | 'DStarEqual', # **=, which bash doesn't have
|
| 363 | 'DSlashEqual', # //=, which bash doesn't have
|
| 364 | 'CastedDummy', # Used for @() $() (words in lex_mode_e.ShCommand)
|
| 365 | # and ${} '' "" (and all other strings)
|
| 366 |
|
| 367 | # Constants
|
| 368 | 'Null',
|
| 369 | 'True',
|
| 370 | 'False',
|
| 371 |
|
| 372 | # Keywords are resolved after lexing, but otherwise behave like tokens.
|
| 373 | 'And',
|
| 374 | 'Or',
|
| 375 | 'Not',
|
| 376 |
|
| 377 | # List comprehensions
|
| 378 | 'For',
|
| 379 | 'Is',
|
| 380 | 'In',
|
| 381 | 'If',
|
| 382 | 'Else',
|
| 383 | 'Capture',
|
| 384 | 'As',
|
| 385 |
|
| 386 | # Unused
|
| 387 | 'Func',
|
| 388 | 'Proc',
|
| 389 | ])
|
| 390 |
|
| 391 | # For C-escaped strings.
|
| 392 | spec.AddKind(
|
| 393 | 'Char',
|
| 394 | [
|
| 395 | 'OneChar',
|
| 396 | 'Stop',
|
| 397 | 'Hex', # \xff
|
| 398 | 'YHex', # \yff for J8 notation
|
| 399 |
|
| 400 | # Two variants of Octal: \377, and \0377.
|
| 401 | 'Octal3',
|
| 402 | 'Octal4',
|
| 403 | 'Unicode4',
|
| 404 | 'SurrogatePair', # JSON
|
| 405 | 'Unicode8', # bash
|
| 406 | 'UBraced',
|
| 407 | 'Pound', # YSH
|
| 408 | 'AsciiControl', # \x01-\x1f, what's disallowed in JSON
|
| 409 | ])
|
| 410 |
|
| 411 | # For lex_mode_e.BashRegex
|
| 412 | # Bash treats ( | ) as special, and space is allowed within ()
|
| 413 | # Note Id.Op_RParen -> Id.Right_BashRegex with lexer hint
|
| 414 | spec.AddKind('BashRegex', ['LParen', 'AllowedInParens'])
|
| 415 |
|
| 416 | spec.AddKind(
|
| 417 | 'Eggex',
|
| 418 | [
|
| 419 | 'Start', # ^ or %start
|
| 420 | 'End', # $ or %end
|
| 421 | 'Dot', # . or dot
|
| 422 | # Future: %boundary generates \b in Python/Perl, etc.
|
| 423 | ])
|
| 424 |
|
| 425 | spec.AddKind(
|
| 426 | 'Redir',
|
| 427 | [
|
| 428 | 'Less', # < stdin
|
| 429 | 'Great', # > stdout
|
| 430 | 'DLess', # << here doc redirect
|
| 431 | 'TLess', # <<< bash only here string
|
| 432 | 'DGreat', # >> append stdout
|
| 433 | 'GreatAnd', # >& descriptor redirect
|
| 434 | 'LessAnd', # <& descriptor redirect
|
| 435 | 'DLessDash', # <<- here doc redirect for tabs?
|
| 436 | 'LessGreat', # <>
|
| 437 | 'Clobber', # >| POSIX?
|
| 438 | 'AndGreat', # bash &> stdout/stderr to file
|
| 439 | 'AndDGreat', # bash &>> stdout/stderr append to file
|
| 440 |
|
| 441 | #'GreatPlus', # >+ is append in YSH
|
| 442 | #'DGreatPlus', # >>+ is append to string in YSH
|
| 443 | ])
|
| 444 |
|
| 445 | # NOTE: This is for left/right WORDS only. (( is not a word so it doesn't
|
| 446 | # get that.
|
| 447 | spec.AddKind(
|
| 448 | 'Left',
|
| 449 | [
|
| 450 | 'DoubleQuote',
|
| 451 | 'JDoubleQuote', # j" for J8 notation
|
| 452 | 'SingleQuote', # ''
|
| 453 | 'DollarSingleQuote', # $'' for \n escapes
|
| 454 | 'RSingleQuote', # r''
|
| 455 | 'USingleQuote', # u''
|
| 456 | 'BSingleQuote', # b''
|
| 457 |
|
| 458 | # Multiline versions
|
| 459 | 'TDoubleQuote', # """ """
|
| 460 | 'DollarTDoubleQuote', # $""" """
|
| 461 | 'TSingleQuote', # ''' '''
|
| 462 | 'RTSingleQuote', # r''' '''
|
| 463 | 'UTSingleQuote', # u''' '''
|
| 464 | 'BTSingleQuote', # b''' '''
|
| 465 | 'Backtick', # `
|
| 466 | 'DollarParen', # $(
|
| 467 | 'DollarBrace', # ${
|
| 468 | 'DollarBraceZsh', # ${(foo)
|
| 469 | 'DollarDParen', # $((
|
| 470 | 'DollarBracket', # $[ - synonym for $(( in bash and zsh
|
| 471 | 'AtBracket', # @[expr] array splice in expression mode
|
| 472 | 'DollarDoubleQuote', # $" for bash localized strings
|
| 473 | 'ProcSubIn', # <( )
|
| 474 | 'ProcSubOut', # >( )
|
| 475 | 'AtParen', # @( for split command sub
|
| 476 | 'CaretParen', # ^( for Block literal in expression mode
|
| 477 | 'CaretBracket', # ^[ for Expr literal
|
| 478 | 'CaretBrace', # ^{ for Arglist
|
| 479 | 'CaretDoubleQuote', # ^" for Template
|
| 480 | 'ColonPipe', # :| for word arrays
|
| 481 | 'PercentParen', # legacy %( for word arrays
|
| 482 | ])
|
| 483 |
|
| 484 | spec.AddKind(
|
| 485 | 'Right',
|
| 486 | [
|
| 487 | 'DoubleQuote',
|
| 488 | 'SingleQuote',
|
| 489 | 'Backtick', # `
|
| 490 | 'DollarBrace', # }
|
| 491 | 'DollarDParen', # )) -- really the second one is a PushHint()
|
| 492 | # ArithSub2 is just Id.Arith_RBracket
|
| 493 | 'DollarDoubleQuote', # "
|
| 494 | 'DollarSingleQuote', # '
|
| 495 |
|
| 496 | # Disambiguated right parens
|
| 497 | 'Subshell', # )
|
| 498 | 'ShFunction', # )
|
| 499 | 'CasePat', # )
|
| 500 | 'Initializer', # )
|
| 501 | 'ExtGlob', # )
|
| 502 | 'BashRegexGroup', # )
|
| 503 | 'BlockLiteral', # } that matches &{ echo hi }
|
| 504 | ])
|
| 505 |
|
| 506 | spec.AddKind('ExtGlob', ['Comma', 'At', 'Star', 'Plus', 'QMark', 'Bang'])
|
| 507 |
|
| 508 | # First position of var sub ${
|
| 509 | # Id.VOp2_Pound -- however you can't tell the difference at first! It could
|
| 510 | # be an op or a name. So it makes sense to base i on the state.
|
| 511 | # Id.VOp2_At
|
| 512 | # But then you have AS_STAR, or Id.Arith_Star maybe
|
| 513 |
|
| 514 | spec.AddKind(
|
| 515 | 'VSub',
|
| 516 | [
|
| 517 | 'DollarName', # $foo
|
| 518 | 'Name', # 'foo' in ${foo}
|
| 519 | 'Number', # $0 .. $9
|
| 520 | 'Bang', # $!
|
| 521 | 'At', # $@ or [@] for array subscripting
|
| 522 | 'Pound', # $# or ${#var} for length
|
| 523 | 'Dollar', # $$
|
| 524 | 'Star', # $*
|
| 525 | 'Hyphen', # $-
|
| 526 | 'QMark', # $?
|
| 527 | 'Dot', # ${.myproc builtin sub}
|
| 528 | ])
|
| 529 |
|
| 530 | spec.AddKindPairs('VTest', [
|
| 531 | ('ColonHyphen', ':-'),
|
| 532 | ('Hyphen', '-'),
|
| 533 | ('ColonEquals', ':='),
|
| 534 | ('Equals', '='),
|
| 535 | ('ColonQMark', ':?'),
|
| 536 | ('QMark', '?'),
|
| 537 | ('ColonPlus', ':+'),
|
| 538 | ('Plus', '+'),
|
| 539 | ])
|
| 540 |
|
| 541 | # Statically parse @P, so @x etc. is an error.
|
| 542 | spec.AddKindPairs(
|
| 543 | 'VOp0',
|
| 544 | [
|
| 545 | ('Q', '@Q'), # ${x@Q} for quoting
|
| 546 | ('E', '@E'),
|
| 547 | ('P', '@P'), # ${PS1@P} for prompt eval
|
| 548 | ('A', '@A'),
|
| 549 | ('a', '@a'),
|
| 550 | ])
|
| 551 |
|
| 552 | # String removal ops
|
| 553 | spec.AddKindPairs(
|
| 554 | 'VOp1',
|
| 555 | [
|
| 556 | ('Percent', '%'),
|
| 557 | ('DPercent', '%%'),
|
| 558 | ('Pound', '#'),
|
| 559 | ('DPound', '##'),
|
| 560 | # Case ops, in bash. At least parse them. Execution might require
|
| 561 | # unicode stuff.
|
| 562 | ('Caret', '^'),
|
| 563 | ('DCaret', '^^'),
|
| 564 | ('Comma', ','),
|
| 565 | ('DComma', ',,'),
|
| 566 | ])
|
| 567 |
|
| 568 | spec.AddKindPairs(
|
| 569 | 'VOpYsh',
|
| 570 | [
|
| 571 | ('Pipe', '|'), # ${x|html}
|
| 572 | ('Space', ' '), # ${x %.3f}
|
| 573 | ])
|
| 574 |
|
| 575 | # Not in POSIX, but in Bash
|
| 576 | spec.AddKindPairs(
|
| 577 | 'VOp2',
|
| 578 | [
|
| 579 | ('Slash', '/'), # / for replacement
|
| 580 | ('Colon', ':'), # : for slicing
|
| 581 | ('LBracket', '['), # [ for indexing
|
| 582 | ('RBracket', ']'), # ] for indexing
|
| 583 | ])
|
| 584 |
|
| 585 | # Can only occur after ${!prefix@}
|
| 586 | spec.AddKindPairs('VOp3', [
|
| 587 | ('At', '@'),
|
| 588 | ('Star', '*'),
|
| 589 | ])
|
| 590 |
|
| 591 | # This kind is for Node types that are NOT tokens.
|
| 592 | spec.AddKind(
|
| 593 | 'Node',
|
| 594 | [
|
| 595 | # Arithmetic nodes
|
| 596 | 'PostDPlus',
|
| 597 | 'PostDMinus', # Postfix inc/dec.
|
| 598 | # Prefix inc/dec use Arith_DPlus/Arith_DMinus.
|
| 599 | 'UnaryPlus',
|
| 600 | 'UnaryMinus', # +1 and -1, to distinguish from infix.
|
| 601 | # Actually we don't need this because we they
|
| 602 | # will be under Expr1/Plus vs Expr2/Plus.
|
| 603 | 'NotIn',
|
| 604 | 'IsNot', # For YSH comparisons
|
| 605 | ])
|
| 606 |
|
| 607 | # NOTE: Not doing AddKindPairs() here because oil will have a different set
|
| 608 | # of keywords. It will probably have for/in/while/until/case/if/else/elif,
|
| 609 | # and then func/proc.
|
| 610 | spec.AddKind(
|
| 611 | 'KW',
|
| 612 | [
|
| 613 | 'DLeftBracket',
|
| 614 | 'Bang',
|
| 615 | 'For',
|
| 616 | 'While',
|
| 617 | 'Until',
|
| 618 | 'Do',
|
| 619 | 'Done',
|
| 620 | 'In',
|
| 621 | 'Case',
|
| 622 | 'Esac',
|
| 623 | 'If',
|
| 624 | 'Fi',
|
| 625 | 'Then',
|
| 626 | 'Else',
|
| 627 | 'Elif',
|
| 628 | 'Function',
|
| 629 | 'Time',
|
| 630 |
|
| 631 | # YSH keywords.
|
| 632 | 'Const',
|
| 633 | 'Var',
|
| 634 | 'SetVar',
|
| 635 | 'SetGlobal',
|
| 636 | # later: Auto?
|
| 637 | 'Call',
|
| 638 | 'Proc',
|
| 639 | 'Typed',
|
| 640 | 'Func',
|
| 641 |
|
| 642 | # builtins, NOT keywords: use, fork, wait, etc.
|
| 643 | # Things that don't affect parsing shouldn't be keywords.
|
| 644 | ])
|
| 645 |
|
| 646 | # Unlike bash, we parse control flow statically. They're not
|
| 647 | # dynamically-resolved builtins.
|
| 648 | spec.AddKind('ControlFlow', ['Break', 'Continue', 'Return', 'Exit'])
|
| 649 |
|
| 650 | # Special Kind for lookahead in the lexer. It's never seen by anything else.
|
| 651 | spec.AddKind('LookAhead', ['FuncParens'])
|
| 652 |
|
| 653 | # For parsing globs and converting them to regexes.
|
| 654 | spec.AddKind('Glob', [
|
| 655 | 'LBracket',
|
| 656 | 'RBracket',
|
| 657 | 'Star',
|
| 658 | 'QMark',
|
| 659 | 'Bang',
|
| 660 | 'Caret',
|
| 661 | 'EscapedChar',
|
| 662 | 'BadBackslash',
|
| 663 | 'CleanLiterals',
|
| 664 | 'OtherLiteral',
|
| 665 | ])
|
| 666 |
|
| 667 | # For C-escaped strings.
|
| 668 | spec.AddKind(
|
| 669 | 'Format',
|
| 670 | [
|
| 671 | 'EscapedPercent',
|
| 672 | 'Percent', # starts another lexer mode
|
| 673 | 'Flag',
|
| 674 | 'Num',
|
| 675 | 'Dot',
|
| 676 | 'Type',
|
| 677 | 'Star',
|
| 678 | 'Time',
|
| 679 | 'Zero',
|
| 680 | ])
|
| 681 |
|
| 682 | # For parsing prompt strings like PS1.
|
| 683 | spec.AddKind('PS', [
|
| 684 | 'Subst',
|
| 685 | 'Octal3',
|
| 686 | 'LBrace',
|
| 687 | 'RBrace',
|
| 688 | 'Literals',
|
| 689 | 'BadBackslash',
|
| 690 | ])
|
| 691 |
|
| 692 | spec.AddKind('Range', ['Int', 'Char', 'Dots', 'Other'])
|
| 693 |
|
| 694 | spec.AddKind(
|
| 695 | 'J8',
|
| 696 | [
|
| 697 | 'LBracket',
|
| 698 | 'RBracket',
|
| 699 | 'LBrace',
|
| 700 | 'RBrace',
|
| 701 | 'Comma',
|
| 702 | 'Colon',
|
| 703 | 'Null',
|
| 704 | 'Bool',
|
| 705 | 'Int', # Number
|
| 706 | 'Float', # Number
|
| 707 |
|
| 708 | # High level tokens for "" b'' u''
|
| 709 | # We don't distinguish them in the parser, because we recognize
|
| 710 | # strings in the lexer.
|
| 711 | 'String',
|
| 712 |
|
| 713 | # JSON8 and NIL8
|
| 714 | 'Identifier',
|
| 715 | 'Newline', # J8 Lines only, similar to Op_Newline
|
| 716 | 'Tab', # Reserved for TSV8
|
| 717 |
|
| 718 | # NIL8 only
|
| 719 | 'LParen',
|
| 720 | 'RParen',
|
| 721 | #'Symbol',
|
| 722 | 'Operator',
|
| 723 | ])
|
| 724 |
|
| 725 | spec.AddKind('ShNumber', ['Dec', 'Hex', 'Oct', 'BaseN'])
|
| 726 |
|
| 727 |
|
| 728 | # Shared between [[ and test/[.
|
| 729 | _UNARY_STR_CHARS = 'zn' # -z -n
|
| 730 | _UNARY_OTHER_CHARS = 'otvR' # -o is overloaded
|
| 731 | _UNARY_PATH_CHARS = 'abcdefghkLprsSuwxOGN' # -a is overloaded
|
| 732 |
|
| 733 | _BINARY_PATH = ['ef', 'nt', 'ot']
|
| 734 | _BINARY_INT = ['eq', 'ne', 'gt', 'ge', 'lt', 'le']
|
| 735 |
|
| 736 |
|
| 737 | def _Dash(strs):
|
| 738 | # type: (List[str]) -> List[Tuple[str, str]]
|
| 739 | # Gives a pair of (token name, string to match)
|
| 740 | return [(s, '-' + s) for s in strs]
|
| 741 |
|
| 742 |
|
| 743 | def AddBoolKinds(spec):
|
| 744 | # type: (IdSpec) -> None
|
| 745 | spec.AddBoolKind('BoolUnary', [
|
| 746 | (bool_arg_type_e.Str, _Dash(list(_UNARY_STR_CHARS))),
|
| 747 | (bool_arg_type_e.Other, _Dash(list(_UNARY_OTHER_CHARS))),
|
| 748 | (bool_arg_type_e.Path, _Dash(list(_UNARY_PATH_CHARS))),
|
| 749 | ])
|
| 750 |
|
| 751 | Id = spec.id_str2int
|
| 752 |
|
| 753 | # test --true and test --false have no single letter flags. They need no
|
| 754 | # lexing.
|
| 755 | for long_flag in ('true', 'false'):
|
| 756 | id_name = 'BoolUnary_%s' % long_flag
|
| 757 | spec._AddId(id_name)
|
| 758 | spec.AddBoolOp(Id[id_name], bool_arg_type_e.Str)
|
| 759 |
|
| 760 | spec.AddBoolKind('BoolBinary', [
|
| 761 | (bool_arg_type_e.Str, [
|
| 762 | ('GlobEqual', '='),
|
| 763 | ('GlobDEqual', '=='),
|
| 764 | ('GlobNEqual', '!='),
|
| 765 | ('EqualTilde', '=~'),
|
| 766 | ]),
|
| 767 | (bool_arg_type_e.Path, _Dash(_BINARY_PATH)),
|
| 768 | (bool_arg_type_e.Int, _Dash(_BINARY_INT)),
|
| 769 | ])
|
| 770 |
|
| 771 | # logical, arity, arg_type
|
| 772 | spec.AddBoolOp(Id['Op_DAmp'], bool_arg_type_e.Undefined)
|
| 773 | spec.AddBoolOp(Id['Op_DPipe'], bool_arg_type_e.Undefined)
|
| 774 | spec.AddBoolOp(Id['KW_Bang'], bool_arg_type_e.Undefined)
|
| 775 |
|
| 776 | spec.AddBoolOp(Id['Op_Less'], bool_arg_type_e.Str)
|
| 777 | spec.AddBoolOp(Id['Op_Great'], bool_arg_type_e.Str)
|
| 778 |
|
| 779 |
|
| 780 | def SetupTestBuiltin(
|
| 781 | id_spec, # type: IdSpec
|
| 782 | unary_lookup, # type: Dict[str, int]
|
| 783 | binary_lookup, # type: Dict[str, int]
|
| 784 | other_lookup, # type: Dict[str, int]
|
| 785 | ):
|
| 786 | # type: (...) -> None
|
| 787 | """Setup tokens for test/[.
|
| 788 |
|
| 789 | Similar to _AddBoolKinds above. Differences:
|
| 790 | - =~ doesn't exist
|
| 791 | - && -> -a, || -> -o
|
| 792 | - ( ) -> Op_LParen (they don't appear above)
|
| 793 | """
|
| 794 | Id = id_spec.id_str2int
|
| 795 | Kind = id_spec.kind_str2int
|
| 796 |
|
| 797 | for letter in _UNARY_STR_CHARS + _UNARY_OTHER_CHARS + _UNARY_PATH_CHARS:
|
| 798 | id_name = 'BoolUnary_%s' % letter
|
| 799 | unary_lookup['-' + letter] = Id[id_name]
|
| 800 |
|
| 801 | for s in _BINARY_PATH + _BINARY_INT:
|
| 802 | id_name = 'BoolBinary_%s' % s
|
| 803 | binary_lookup['-' + s] = Id[id_name]
|
| 804 |
|
| 805 | # Like the [[ definition above, but without globbing and without =~ .
|
| 806 |
|
| 807 | for id_name, token_str in [('Equal', '='), ('DEqual', '=='),
|
| 808 | ('NEqual', '!=')]:
|
| 809 | id_int = id_spec.AddBoolBinaryForBuiltin(id_name, Kind['BoolBinary'])
|
| 810 |
|
| 811 | binary_lookup[token_str] = id_int
|
| 812 |
|
| 813 | # Some of these names don't quite match, but it keeps the BoolParser simple.
|
| 814 | binary_lookup['<'] = Id['Op_Less']
|
| 815 | binary_lookup['>'] = Id['Op_Great']
|
| 816 |
|
| 817 | # NOTE: -a and -o overloaded as unary prefix operators BoolUnary_a and
|
| 818 | # BoolUnary_o. The parser rather than the tokenizer handles this.
|
| 819 | other_lookup['!'] = Id['KW_Bang'] # like [[ !
|
| 820 | other_lookup['('] = Id['Op_LParen']
|
| 821 | other_lookup[')'] = Id['Op_RParen']
|
| 822 |
|
| 823 | other_lookup[']'] = Id['Arith_RBracket'] # For closing ]
|