OILS / frontend / args.py View on Github | oilshell.org

677 lines, 350 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from frontend import match
61from mycpp import mops
62from mycpp.mylib import log, tagswitch, iteritems
63
64_ = log
65
66from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
67if TYPE_CHECKING:
68 from frontend import flag_spec
69 OptChange = Tuple[str, bool]
70
71# TODO: Move to flag_spec? We use flag_type_t
72String = 1
73Int = 2
74Float = 3 # e.g. for read -t timeout value
75Bool = 4
76
77
78class _Attributes(object):
79 """Object to hold flags.
80
81 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
82 """
83
84 def __init__(self, defaults):
85 # type: (Dict[str, value_t]) -> None
86
87 # New style
88 self.attrs = {} # type: Dict[str, value_t]
89
90 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
91 self.shopt_changes = [
92 ] # type: List[OptChange] # -O nullglob +O nullglob
93 self.show_options = False # 'set -o' without an argument
94 self.actions = [] # type: List[str] # for compgen -A
95 self.saw_double_dash = False # for set --
96 for name, v in iteritems(defaults):
97 self.Set(name, v)
98
99 def SetTrue(self, name):
100 # type: (str) -> None
101 self.Set(name, value.Bool(True))
102
103 def Set(self, name, val):
104 # type: (str, value_t) -> None
105
106 # debug-completion -> debug_completion
107 name = name.replace('-', '_')
108
109 # similar hack to avoid C++ keyword in frontend/flag_gen.py
110 if name == 'extern':
111 name = 'extern_'
112
113 self.attrs[name] = val
114
115 if 0:
116 # Backward compatibility!
117 with tagswitch(val) as case:
118 if case(value_e.Undef):
119 py_val = None # type: Any
120 elif case(value_e.Bool):
121 py_val = cast(value.Bool, val).b
122 elif case(value_e.Int):
123 py_val = cast(value.Int, val).i
124 elif case(value_e.Float):
125 py_val = cast(value.Float, val).f
126 elif case(value_e.Str):
127 py_val = cast(value.Str, val).s
128 else:
129 raise AssertionError(val)
130
131 setattr(self, name, py_val)
132
133 def __repr__(self):
134 # type: () -> str
135 return '<_Attributes %s>' % self.__dict__
136
137
138class Reader(object):
139 """Wrapper for argv.
140
141 Modified by both the parsing loop and various actions.
142
143 The caller of the flags parser can continue to use it after flag parsing is
144 done to get args.
145 """
146
147 def __init__(self, argv, locs=None):
148 # type: (List[str], Optional[List[CompoundWord]]) -> None
149 self.argv = argv
150 self.locs = locs
151 self.n = len(argv)
152 self.i = 0
153
154 def __repr__(self):
155 # type: () -> str
156 return '<args.Reader %r %d>' % (self.argv, self.i)
157
158 def Next(self):
159 # type: () -> None
160 """Advance."""
161 self.i += 1
162
163 def Peek(self):
164 # type: () -> Optional[str]
165 """Return the next token, or None if there are no more.
166
167 None is your SENTINEL for parsing.
168 """
169 if self.i >= self.n:
170 return None
171 else:
172 return self.argv[self.i]
173
174 def Peek2(self):
175 # type: () -> Tuple[Optional[str], loc_t]
176 """Return the next token, or None if there are no more.
177
178 None is your SENTINEL for parsing.
179 """
180 if self.i >= self.n:
181 return None, loc.Missing
182 else:
183 return self.argv[self.i], self.locs[self.i]
184
185 def ReadRequired(self, error_msg):
186 # type: (str) -> str
187 arg = self.Peek()
188 if arg is None:
189 # point at argv[0]
190 e_usage(error_msg, self._FirstLocation())
191 self.Next()
192 return arg
193
194 def ReadRequired2(self, error_msg):
195 # type: (str) -> Tuple[str, loc_t]
196 arg = self.Peek()
197 if arg is None:
198 # point at argv[0]
199 e_usage(error_msg, self._FirstLocation())
200 location = self.locs[self.i]
201 self.Next()
202 return arg, location
203
204 def Rest(self):
205 # type: () -> List[str]
206 """Return the rest of the arguments."""
207 return self.argv[self.i:]
208
209 def Rest2(self):
210 # type: () -> Tuple[List[str], List[CompoundWord]]
211 """Return the rest of the arguments."""
212 return self.argv[self.i:], self.locs[self.i:]
213
214 def AtEnd(self):
215 # type: () -> bool
216 return self.i >= self.n # must be >= and not ==
217
218 def Done(self):
219 # type: () -> None
220 if not self.AtEnd():
221 e_usage('got too many arguments', self.Location())
222
223 def _FirstLocation(self):
224 # type: () -> loc_t
225 if self.locs is not None and self.locs[0] is not None:
226 return self.locs[0]
227 else:
228 return loc.Missing
229
230 def Location(self):
231 # type: () -> loc_t
232 if self.locs is not None:
233 if self.i == self.n:
234 i = self.n - 1 # if the last arg is missing, point at the one before
235 else:
236 i = self.i
237 if self.locs[i] is not None:
238 return self.locs[i]
239 else:
240 return loc.Missing
241 else:
242 return loc.Missing
243
244
245class _Action(object):
246 """What is done when a flag or option is detected."""
247
248 def __init__(self):
249 # type: () -> None
250 """Empty constructor for mycpp."""
251 pass
252
253 def OnMatch(self, attached_arg, arg_r, out):
254 # type: (Optional[str], Reader, _Attributes) -> bool
255 """Called when the flag matches.
256
257 Args:
258 prefix: '-' or '+'
259 suffix: ',' for -d,
260 arg_r: Reader() (rename to Input or InputReader?)
261 out: _Attributes() -- the thing we want to set
262
263 Returns:
264 True if flag parsing should be aborted.
265 """
266 raise NotImplementedError()
267
268
269class _ArgAction(_Action):
270
271 def __init__(self, name, quit_parsing_flags, valid=None):
272 # type: (str, bool, Optional[List[str]]) -> None
273 """
274 Args:
275 quit_parsing_flags: Stop parsing args after this one. for sh -c.
276 python -c behaves the same way.
277 """
278 self.name = name
279 self.quit_parsing_flags = quit_parsing_flags
280 self.valid = valid
281
282 def _Value(self, arg, location):
283 # type: (str, loc_t) -> value_t
284 raise NotImplementedError()
285
286 def OnMatch(self, attached_arg, arg_r, out):
287 # type: (Optional[str], Reader, _Attributes) -> bool
288 """Called when the flag matches."""
289 if attached_arg is not None: # for the ',' in -d,
290 arg = attached_arg
291 else:
292 arg_r.Next()
293 arg = arg_r.Peek()
294 if arg is None:
295 e_usage('expected argument to %r' % ('-' + self.name),
296 arg_r.Location())
297
298 val = self._Value(arg, arg_r.Location())
299 out.Set(self.name, val)
300 return self.quit_parsing_flags
301
302
303class SetToInt(_ArgAction):
304
305 def __init__(self, name):
306 # type: (str) -> None
307 # repeat defaults for C++ translation
308 _ArgAction.__init__(self, name, False, valid=None)
309
310 def _Value(self, arg, location):
311 # type: (str, loc_t) -> value_t
312 if match.LooksLikeInteger(arg):
313 ok, i = mops.FromStr2(arg)
314 if not ok:
315 e_usage('Integer too big: %s' % arg, location)
316 else:
317 e_usage(
318 'expected integer after %s, got %r' % ('-' + self.name, arg),
319 location)
320
321 # So far all our int values are > 0, so use -1 as the 'unset' value
322 # corner case: this treats -0 as 0!
323 if mops.Greater(mops.BigInt(0), i):
324 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
325 location)
326 return value.Int(i)
327
328
329class SetToFloat(_ArgAction):
330
331 def __init__(self, name):
332 # type: (str) -> None
333 # repeat defaults for C++ translation
334 _ArgAction.__init__(self, name, False, valid=None)
335
336 def _Value(self, arg, location):
337 # type: (str, loc_t) -> value_t
338 try:
339 f = float(arg)
340 except ValueError:
341 e_usage(
342 'expected number after %r, got %r' % ('-' + self.name, arg),
343 location)
344 # So far all our float values are > 0, so use -1.0 as the 'unset' value
345 # corner case: this treats -0.0 as 0.0!
346 if f < 0:
347 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
348 location)
349 return value.Float(f)
350
351
352class SetToString(_ArgAction):
353
354 def __init__(self, name, quit_parsing_flags, valid=None):
355 # type: (str, bool, Optional[List[str]]) -> None
356 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
357
358 def _Value(self, arg, location):
359 # type: (str, loc_t) -> value_t
360 if self.valid is not None and arg not in self.valid:
361 e_usage(
362 'got invalid argument %r to %r, expected one of: %s' %
363 (arg, ('-' + self.name), '|'.join(self.valid)), location)
364 return value.Str(arg)
365
366
367class SetAttachedBool(_Action):
368
369 def __init__(self, name):
370 # type: (str) -> None
371 self.name = name
372
373 def OnMatch(self, attached_arg, arg_r, out):
374 # type: (Optional[str], Reader, _Attributes) -> bool
375 """Called when the flag matches."""
376
377 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
378 #
379 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
380 #
381 # 'attached_arg' is also used for -t0 though, which is weird
382
383 if attached_arg is not None: # '0' in --verbose=0
384 if attached_arg in ('0', 'F', 'false',
385 'False'): # TODO: incorrect translation
386 b = False
387 elif attached_arg in ('1', 'T', 'true', 'Talse'):
388 b = True
389 else:
390 e_usage(
391 'got invalid argument to boolean flag: %r' % attached_arg,
392 loc.Missing)
393 else:
394 b = True
395
396 out.Set(self.name, value.Bool(b))
397 return False
398
399
400class SetToTrue(_Action):
401
402 def __init__(self, name):
403 # type: (str) -> None
404 self.name = name
405
406 def OnMatch(self, attached_arg, arg_r, out):
407 # type: (Optional[str], Reader, _Attributes) -> bool
408 """Called when the flag matches."""
409 out.SetTrue(self.name)
410 return False
411
412
413class SetOption(_Action):
414 """Set an option to a boolean, for 'set +e'."""
415
416 def __init__(self, name):
417 # type: (str) -> None
418 self.name = name
419
420 def OnMatch(self, attached_arg, arg_r, out):
421 # type: (Optional[str], Reader, _Attributes) -> bool
422 """Called when the flag matches."""
423 b = (attached_arg == '-')
424 out.opt_changes.append((self.name, b))
425 return False
426
427
428class SetNamedOption(_Action):
429 """Set a named option to a boolean, for 'set +o errexit'."""
430
431 def __init__(self, shopt=False):
432 # type: (bool) -> None
433 self.names = [] # type: List[str]
434 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
435
436 def ArgName(self, name):
437 # type: (str) -> None
438 self.names.append(name)
439
440 def OnMatch(self, attached_arg, arg_r, out):
441 # type: (Optional[str], Reader, _Attributes) -> bool
442 """Called when the flag matches."""
443 b = (attached_arg == '-')
444 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
445 arg_r.Next() # always advance
446 arg = arg_r.Peek()
447 if arg is None:
448 # triggers on 'set -O' in addition to 'set -o' (meh OK)
449 out.show_options = True
450 return True # quit parsing
451
452 attr_name = arg # Note: validation is done elsewhere
453 if len(self.names) and attr_name not in self.names:
454 e_usage('Invalid option %r' % arg, loc.Missing)
455 changes = out.shopt_changes if self.shopt else out.opt_changes
456 changes.append((attr_name, b))
457 return False
458
459
460class SetAction(_Action):
461 """For compgen -f."""
462
463 def __init__(self, name):
464 # type: (str) -> None
465 self.name = name
466
467 def OnMatch(self, attached_arg, arg_r, out):
468 # type: (Optional[str], Reader, _Attributes) -> bool
469 out.actions.append(self.name)
470 return False
471
472
473class SetNamedAction(_Action):
474 """For compgen -A file."""
475
476 def __init__(self):
477 # type: () -> None
478 self.names = [] # type: List[str]
479
480 def ArgName(self, name):
481 # type: (str) -> None
482 self.names.append(name)
483
484 def OnMatch(self, attached_arg, arg_r, out):
485 # type: (Optional[str], Reader, _Attributes) -> bool
486 """Called when the flag matches."""
487 arg_r.Next() # always advance
488 arg = arg_r.Peek()
489 if arg is None:
490 e_usage('Expected argument for action', loc.Missing)
491
492 attr_name = arg
493 # Validate the option name against a list of valid names.
494 if len(self.names) and attr_name not in self.names:
495 e_usage('Invalid action name %r' % arg, loc.Missing)
496 out.actions.append(attr_name)
497 return False
498
499
500def Parse(spec, arg_r):
501 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
502
503 # NOTE about -:
504 # 'set -' ignores it, vs set
505 # 'unset -' or 'export -' seems to treat it as a variable name
506 out = _Attributes(spec.defaults)
507
508 while not arg_r.AtEnd():
509 arg = arg_r.Peek()
510 if arg == '--':
511 out.saw_double_dash = True
512 arg_r.Next()
513 break
514
515 # Only accept -- if there are any long flags defined
516 if len(spec.actions_long) and arg.startswith('--'):
517 pos = arg.find('=', 2)
518 if pos == -1:
519 suffix = None # type: Optional[str]
520 flag_name = arg[2:] # strip off --
521 else:
522 suffix = arg[pos + 1:]
523 flag_name = arg[2:pos]
524
525 action = spec.actions_long.get(flag_name)
526 if action is None:
527 e_usage('got invalid flag %r' % arg, arg_r.Location())
528
529 action.OnMatch(suffix, arg_r, out)
530 arg_r.Next()
531 continue
532
533 elif arg.startswith('-') and len(arg) > 1:
534 n = len(arg)
535 for i in xrange(1, n): # parse flag combos like -rx
536 ch = arg[i]
537
538 if ch == '0':
539 ch = 'Z' # hack for read -0
540
541 if ch in spec.plus_flags:
542 out.Set(ch, value.Str('-'))
543 continue
544
545 if ch in spec.arity0: # e.g. read -r
546 out.SetTrue(ch)
547 continue
548
549 if ch in spec.arity1: # e.g. read -t1.0
550 action = spec.arity1[ch]
551 # make sure we don't pass empty string for read -t
552 attached_arg = arg[i + 1:] if i < n - 1 else None
553 action.OnMatch(attached_arg, arg_r, out)
554 break
555
556 e_usage("doesn't accept flag %s" % ('-' + ch),
557 arg_r.Location())
558
559 arg_r.Next() # next arg
560
561 # Only accept + if there are ANY options defined, e.g. for declare +rx.
562 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
563 n = len(arg)
564 for i in xrange(1, n): # parse flag combos like -rx
565 ch = arg[i]
566 if ch in spec.plus_flags:
567 out.Set(ch, value.Str('+'))
568 continue
569
570 e_usage("doesn't accept option %s" % ('+' + ch),
571 arg_r.Location())
572
573 arg_r.Next() # next arg
574
575 else: # a regular arg
576 break
577
578 return out
579
580
581def ParseLikeEcho(spec, arg_r):
582 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
583 """Echo is a special case. These work: echo -n echo -en.
584
585 - But don't respect --
586 - doesn't fail when an invalid flag is passed
587 """
588 out = _Attributes(spec.defaults)
589
590 while not arg_r.AtEnd():
591 arg = arg_r.Peek()
592 chars = arg[1:]
593 if arg.startswith('-') and len(chars):
594 # Check if it looks like -en or not. TODO: could optimize this.
595 done = False
596 for c in chars:
597 if c not in spec.arity0:
598 done = True
599 break
600 if done:
601 break
602
603 for ch in chars:
604 out.SetTrue(ch)
605
606 else:
607 break # Looks like an arg
608
609 arg_r.Next() # next arg
610
611 return out
612
613
614def ParseMore(spec, arg_r):
615 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
616 """Return attributes and an index.
617
618 Respects +, like set +eu
619
620 We do NOT respect:
621
622 WRONG: sh -cecho OK: sh -c echo
623 WRONG: set -opipefail OK: set -o pipefail
624
625 But we do accept these
626
627 set -euo pipefail
628 set -oeu pipefail
629 set -oo pipefail errexit
630 """
631 out = _Attributes(spec.defaults)
632
633 quit = False
634 while not arg_r.AtEnd():
635 arg = arg_r.Peek()
636 if arg == '--':
637 out.saw_double_dash = True
638 arg_r.Next()
639 break
640
641 if arg.startswith('--'):
642 action = spec.actions_long.get(arg[2:])
643 if action is None:
644 e_usage('got invalid flag %r' % arg, arg_r.Location())
645
646 # Note: not parsing --foo=bar as attached_arg, as above
647 action.OnMatch(None, arg_r, out)
648 arg_r.Next()
649 continue
650
651 # corner case: sh +c is also accepted!
652 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
653 # note: we're not handling sh -cecho (no space) as an argument
654 # It complains about a missing argument
655
656 char0 = arg[0]
657
658 # TODO: set - - empty
659 for ch in arg[1:]:
660 #log('ch %r arg_r %s', ch, arg_r)
661 action = spec.actions_short.get(ch)
662 if action is None:
663 e_usage('got invalid flag %r' % ('-' + ch),
664 arg_r.Location())
665
666 attached_arg = char0 if ch in spec.plus_flags else None
667 quit = action.OnMatch(attached_arg, arg_r, out)
668 arg_r.Next() # process the next flag
669
670 if quit:
671 break
672 else:
673 continue
674
675 break # it's a regular arg
676
677 return out