OILS / frontend / args.py View on Github | oils.pub

718 lines, 364 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, iteritems
62
63_ = log
64
65from typing import (Tuple, Optional, Dict, List, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 # -o errexit +o nounset
90 self.opt_changes = [] # type: List[OptChange]
91
92 # -O nullglob +O nullglob
93 self.shopt_changes = [] # type: List[OptChange]
94
95 # Special case for --eval and --eval-pure? For bin/osh. It seems OK
96 # to have one now. The pool tells us if it was pure or not.
97 # Note: MAIN_SPEC is different than SET_SPEC, so set --eval does
98 # nothing.
99 self.eval_flags = [] # type: List[Tuple[str, bool]]
100
101 self.show_options = False # 'set -o' without an argument
102 self.actions = [] # type: List[str] # for compgen -A
103 self.saw_double_dash = False # for set --
104 self.saw_single_dash = False # for set -
105 for name, v in iteritems(defaults):
106 self.Set(name, v)
107
108 def SetTrue(self, name):
109 # type: (str) -> None
110 self.Set(name, value.Bool(True))
111
112 def Set(self, name, val):
113 # type: (str, value_t) -> None
114
115 # debug-completion -> debug_completion
116 name = name.replace('-', '_')
117
118 # similar hack to avoid C++ keyword in frontend/flag_gen.py
119 if name == 'extern':
120 name = 'extern_'
121 elif name == 'private':
122 name = 'private_'
123
124 self.attrs[name] = val
125
126 def __repr__(self):
127 # type: () -> str
128 return '<_Attributes %s>' % self.__dict__
129
130
131class Reader(object):
132 """Wrapper for argv.
133
134 Modified by both the parsing loop and various actions.
135
136 The caller of the flags parser can continue to use it after flag parsing is
137 done to get args.
138 """
139
140 def __init__(self, argv, locs=None):
141 # type: (List[str], Optional[List[CompoundWord]]) -> None
142 self.argv = argv
143 self.locs = locs
144 self.n = len(argv)
145 self.i = 0
146
147 def __repr__(self):
148 # type: () -> str
149 return '<args.Reader %r %d>' % (self.argv, self.i)
150
151 def Next(self):
152 # type: () -> None
153 """Advance."""
154 self.i += 1
155
156 def Peek(self):
157 # type: () -> Optional[str]
158 """Return the next token, or None if there are no more.
159
160 None is your SENTINEL for parsing.
161 """
162 if self.i >= self.n:
163 return None
164 else:
165 return self.argv[self.i]
166
167 def Peek2(self):
168 # type: () -> Tuple[Optional[str], loc_t]
169 """Return the next token, or None if there are no more.
170
171 None is your SENTINEL for parsing.
172 """
173 if self.i >= self.n:
174 return None, loc.Missing
175 else:
176 return self.argv[self.i], self.locs[self.i]
177
178 def ReadRequired(self, error_msg):
179 # type: (str) -> str
180 arg = self.Peek()
181 if arg is None:
182 # point at argv[0]
183 e_usage(error_msg, self._FirstLocation())
184 self.Next()
185 return arg
186
187 def ReadRequired2(self, error_msg):
188 # type: (str) -> Tuple[str, CompoundWord]
189 arg = self.Peek()
190 if arg is None:
191 # point at argv[0]
192 e_usage(error_msg, self._FirstLocation())
193 location = self.locs[self.i]
194 self.Next()
195 return arg, location
196
197 def Rest(self):
198 # type: () -> List[str]
199 """Return the rest of the arguments."""
200 return self.argv[self.i:]
201
202 def Rest2(self):
203 # type: () -> Tuple[List[str], List[CompoundWord]]
204 """Return the rest of the arguments."""
205 return self.argv[self.i:], self.locs[self.i:]
206
207 def AtEnd(self):
208 # type: () -> bool
209 return self.i >= self.n # must be >= and not ==
210
211 def Done(self):
212 # type: () -> None
213 if not self.AtEnd():
214 e_usage('got too many arguments', self.Location())
215
216 def _FirstLocation(self):
217 # type: () -> loc_t
218 if self.locs is not None and self.locs[0] is not None:
219 return self.locs[0]
220 else:
221 return loc.Missing
222
223 def Location(self):
224 # type: () -> loc_t
225 if self.locs is not None:
226 if self.i == self.n:
227 i = self.n - 1 # if the last arg is missing, point at the one before
228 else:
229 i = self.i
230 if self.locs[i] is not None:
231 return self.locs[i]
232 else:
233 return loc.Missing
234 else:
235 return loc.Missing
236
237
238class _Action(object):
239 """What is done when a flag or option is detected."""
240
241 def __init__(self):
242 # type: () -> None
243 """Empty constructor for mycpp."""
244 pass
245
246 def OnMatch(self, attached_arg, arg_r, out):
247 # type: (Optional[str], Reader, _Attributes) -> bool
248 """Called when the flag matches.
249
250 Args:
251 prefix: '-' or '+'
252 suffix: ',' for -d,
253 arg_r: Reader() (rename to Input or InputReader?)
254 out: _Attributes() -- the thing we want to set
255
256 Returns:
257 True if flag parsing should be aborted.
258 """
259 raise NotImplementedError()
260
261
262class AppendEvalFlag(_Action):
263
264 def __init__(self, name):
265 # type: (str) -> None
266 _Action.__init__(self)
267 self.name = name
268 self.is_pure = (name == 'eval-pure')
269
270 def OnMatch(self, attached_arg, arg_r, out):
271 # type: (Optional[str], Reader, _Attributes) -> bool
272 """Called when the flag matches."""
273
274 assert attached_arg is None, attached_arg
275
276 arg_r.Next()
277 arg = arg_r.Peek()
278 if arg is None:
279 e_usage('expected argument to %r' % ('--' + self.name),
280 arg_r.Location())
281
282 # is_pure is True for --eval-pure
283 out.eval_flags.append((arg, self.is_pure))
284 return False
285
286
287class _ArgAction(_Action):
288
289 def __init__(self, name, quit_parsing_flags, valid=None):
290 # type: (str, bool, Optional[List[str]]) -> None
291 """
292 Args:
293 quit_parsing_flags: Stop parsing args after this one. for sh -c.
294 python -c behaves the same way.
295 """
296 self.name = name
297 self.quit_parsing_flags = quit_parsing_flags
298 self.valid = valid
299
300 def _Value(self, arg, location):
301 # type: (str, loc_t) -> value_t
302 raise NotImplementedError()
303
304 def OnMatch(self, attached_arg, arg_r, out):
305 # type: (Optional[str], Reader, _Attributes) -> bool
306 """Called when the flag matches."""
307 if attached_arg is not None: # for the ',' in -d,
308 arg = attached_arg
309 else:
310 arg_r.Next()
311 arg = arg_r.Peek()
312 if arg is None:
313 e_usage("expected argument to '-%s'" % self.name,
314 arg_r.Location())
315
316 if self.quit_parsing_flags:
317 # Weird special case for -c - and -c --
318 # This is bug #2638. See spec/sh-usage.test.sh
319 if arg in ('-', '--'):
320 arg_r.Next()
321 arg = arg_r.Peek()
322 if arg is None:
323 e_usage("expected argument to '-%s'" % self.name,
324 arg_r.Location())
325
326 val = self._Value(arg, arg_r.Location())
327 out.Set(self.name, val)
328 return self.quit_parsing_flags
329
330
331class SetToInt(_ArgAction):
332
333 def __init__(self, name):
334 # type: (str) -> None
335 # repeat defaults for C++ translation
336 _ArgAction.__init__(self, name, False, valid=None)
337
338 def _Value(self, arg, location):
339 # type: (str, loc_t) -> value_t
340 #if match.LooksLikeInteger(arg):
341 if True: # break dependency for prebuilt/
342 ok, i = mops.FromStr2(arg)
343 if not ok:
344 #e_usage('Integer too big: %s' % arg, location)
345 e_usage(
346 'expected integer after %s, got %r' %
347 ('-' + self.name, arg), location)
348 else:
349 pass
350 #e_usage(
351 #'expected integer after %s, got %r' % ('-' + self.name, arg),
352 #location)
353
354 # So far all our int values are > 0, so use -1 as the 'unset' value
355 # corner case: this treats -0 as 0!
356 if mops.Greater(mops.BigInt(0), i):
357 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
358 location)
359 return value.Int(i)
360
361
362class SetToFloat(_ArgAction):
363
364 def __init__(self, name):
365 # type: (str) -> None
366 # repeat defaults for C++ translation
367 _ArgAction.__init__(self, name, False, valid=None)
368
369 def _Value(self, arg, location):
370 # type: (str, loc_t) -> value_t
371 try:
372 f = float(arg)
373 except ValueError:
374 e_usage(
375 'expected number after %r, got %r' % ('-' + self.name, arg),
376 location)
377 # So far all our float values are > 0, so use -1.0 as the 'unset' value
378 # corner case: this treats -0.0 as 0.0!
379 if f < 0:
380 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
381 location)
382 return value.Float(f)
383
384
385class SetToString(_ArgAction):
386
387 def __init__(self, name, quit_parsing_flags, valid=None):
388 # type: (str, bool, Optional[List[str]]) -> None
389 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
390
391 def _Value(self, arg, location):
392 # type: (str, loc_t) -> value_t
393 if self.valid is not None and arg not in self.valid:
394 e_usage(
395 'got invalid argument %r to %r, expected one of: %s' %
396 (arg, ('-' + self.name), '|'.join(self.valid)), location)
397 return value.Str(arg)
398
399
400class SetAttachedBool(_Action):
401
402 def __init__(self, name):
403 # type: (str) -> None
404 self.name = name
405
406 def OnMatch(self, attached_arg, arg_r, out):
407 # type: (Optional[str], Reader, _Attributes) -> bool
408 """Called when the flag matches."""
409
410 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
411 #
412 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
413 #
414 # 'attached_arg' is also used for -t0 though, which is weird
415
416 if attached_arg is not None: # '0' in --verbose=0
417 if attached_arg in ('0', 'F', 'false', 'False'):
418 b = False
419 elif attached_arg in ('1', 'T', 'true', 'True'):
420 b = True
421 else:
422 e_usage(
423 'got invalid argument to boolean flag: %r' % attached_arg,
424 loc.Missing)
425 else:
426 b = True
427
428 out.Set(self.name, value.Bool(b))
429 return False
430
431
432class SetToTrue(_Action):
433
434 def __init__(self, name):
435 # type: (str) -> None
436 self.name = name
437
438 def OnMatch(self, attached_arg, arg_r, out):
439 # type: (Optional[str], Reader, _Attributes) -> bool
440 """Called when the flag matches."""
441 out.SetTrue(self.name)
442 return False
443
444
445class SetOption(_Action):
446 """Set an option to a boolean, for 'set +e'."""
447
448 def __init__(self, name):
449 # type: (str) -> None
450 self.name = name
451
452 def OnMatch(self, attached_arg, arg_r, out):
453 # type: (Optional[str], Reader, _Attributes) -> bool
454 """Called when the flag matches."""
455 b = (attached_arg == '-')
456 out.opt_changes.append((self.name, b))
457 return False
458
459
460class SetNamedOption(_Action):
461 """Set a named option to a boolean, for 'set +o errexit'."""
462
463 def __init__(self, shopt=False):
464 # type: (bool) -> None
465 self.names = [] # type: List[str]
466 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
467
468 def ArgName(self, name):
469 # type: (str) -> None
470 self.names.append(name)
471
472 def OnMatch(self, attached_arg, arg_r, out):
473 # type: (Optional[str], Reader, _Attributes) -> bool
474 """Called when the flag matches."""
475 b = (attached_arg == '-')
476 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
477 arg_r.Next() # always advance
478 arg = arg_r.Peek()
479 if arg is None:
480 # triggers on 'set -O' in addition to 'set -o' (meh OK)
481 out.show_options = True
482 return True # quit parsing
483
484 attr_name = arg # Note: validation is done elsewhere
485 if len(self.names) and attr_name not in self.names:
486 e_usage('Invalid option %r' % arg, loc.Missing)
487 changes = out.shopt_changes if self.shopt else out.opt_changes
488 changes.append((attr_name, b))
489 return False
490
491
492class SetAction(_Action):
493 """For compgen -f."""
494
495 def __init__(self, name):
496 # type: (str) -> None
497 self.name = name
498
499 def OnMatch(self, attached_arg, arg_r, out):
500 # type: (Optional[str], Reader, _Attributes) -> bool
501 out.actions.append(self.name)
502 return False
503
504
505class SetNamedAction(_Action):
506 """For compgen -A file."""
507
508 def __init__(self):
509 # type: () -> None
510 self.names = [] # type: List[str]
511
512 def ArgName(self, name):
513 # type: (str) -> None
514 self.names.append(name)
515
516 def OnMatch(self, attached_arg, arg_r, out):
517 # type: (Optional[str], Reader, _Attributes) -> bool
518 """Called when the flag matches."""
519 arg_r.Next() # always advance
520 arg = arg_r.Peek()
521 if arg is None:
522 e_usage('Expected argument for action', loc.Missing)
523
524 attr_name = arg
525 # Validate the option name against a list of valid names.
526 if len(self.names) and attr_name not in self.names:
527 e_usage('Invalid action name %r' % arg, loc.Missing)
528 out.actions.append(attr_name)
529 return False
530
531
532def Parse(spec, arg_r):
533 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
534
535 # NOTE about -:
536 # 'set -' ignores it, vs set
537 # 'unset -' or 'export -' seems to treat it as a variable name
538 out = _Attributes(spec.defaults)
539
540 while not arg_r.AtEnd():
541 arg = arg_r.Peek()
542 if arg == '--':
543 out.saw_double_dash = True
544 arg_r.Next()
545 break
546
547 # Only accept -- if there are any long flags defined
548 if len(spec.actions_long) and arg.startswith('--'):
549 pos = arg.find('=', 2)
550 if pos == -1:
551 suffix = None # type: Optional[str]
552 flag_name = arg[2:] # strip off --
553 else:
554 suffix = arg[pos + 1:]
555 flag_name = arg[2:pos]
556
557 action = spec.actions_long.get(flag_name)
558 if action is None:
559 e_usage('got invalid flag %r' % arg, arg_r.Location())
560
561 action.OnMatch(suffix, arg_r, out)
562 arg_r.Next()
563 continue
564
565 elif arg.startswith('-') and len(arg) > 1:
566 n = len(arg)
567 for i in xrange(1, n): # parse flag combos like -rx
568 ch = arg[i]
569
570 if ch == '0':
571 ch = 'Z' # hack for read -0
572
573 if ch in spec.plus_flags:
574 out.Set(ch, value.Str('-'))
575 continue
576
577 if ch in spec.arity0: # e.g. read -r
578 out.SetTrue(ch)
579 continue
580
581 if ch in spec.arity1: # e.g. read -t1.0
582 action = spec.arity1[ch]
583 # make sure we don't pass empty string for read -t
584 attached_arg = arg[i + 1:] if i < n - 1 else None
585 action.OnMatch(attached_arg, arg_r, out)
586 break
587
588 e_usage("doesn't accept flag %s" % ('-' + ch),
589 arg_r.Location())
590
591 arg_r.Next() # next arg
592
593 # Only accept + if there are ANY options defined, e.g. for declare +rx.
594 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
595 n = len(arg)
596 for i in xrange(1, n): # parse flag combos like -rx
597 ch = arg[i]
598 if ch in spec.plus_flags:
599 out.Set(ch, value.Str('+'))
600 continue
601
602 e_usage("doesn't accept option %s" % ('+' + ch),
603 arg_r.Location())
604
605 arg_r.Next() # next arg
606
607 else: # a regular arg
608 break
609
610 return out
611
612
613def ParseLikeEcho(spec, arg_r):
614 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
615 """Echo is a special case. These work: echo -n echo -en.
616
617 - But don't respect --
618 - doesn't fail when an invalid flag is passed
619 """
620 out = _Attributes(spec.defaults)
621
622 while not arg_r.AtEnd():
623 arg = arg_r.Peek()
624 chars = arg[1:]
625 if arg.startswith('-') and len(chars):
626 # Check if it looks like -en or not. TODO: could optimize this.
627 done = False
628 for c in chars:
629 if c not in spec.arity0:
630 done = True
631 break
632 if done:
633 break
634
635 for ch in chars:
636 out.SetTrue(ch)
637
638 else:
639 break # Looks like an arg
640
641 arg_r.Next() # next arg
642
643 return out
644
645
646def ParseMore(spec, arg_r):
647 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
648 """Return attributes and an index.
649
650 Respects +, like set +eu
651
652 We do NOT respect:
653
654 WRONG: sh -cecho OK: sh -c echo
655 WRONG: set -opipefail OK: set -o pipefail
656
657 But we do accept these
658
659 set -euo pipefail
660 set -oeu pipefail
661 set -oo pipefail errexit
662 """
663 out = _Attributes(spec.defaults)
664
665 quit = False
666 while not arg_r.AtEnd():
667 arg = arg_r.Peek()
668 if arg == '--':
669 out.saw_double_dash = True
670 arg_r.Next()
671 break
672
673 if arg == '-': # weird special behavior for 'set -'
674 out.saw_single_dash = True
675 arg_r.Next()
676 break
677
678 if arg == '+': # a single + is an ignored flag
679 arg_r.Next()
680 continue
681
682 if arg.startswith('--'):
683 action = spec.actions_long.get(arg[2:])
684 if action is None:
685 e_usage('got invalid flag %r' % arg, arg_r.Location())
686
687 # Note: not parsing --foo=bar as attached_arg, as above
688 action.OnMatch(None, arg_r, out)
689 arg_r.Next()
690 continue
691
692 # corner case: sh +c is also accepted!
693 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
694 # note: we're not handling sh -cecho (no space) as an argument
695 # It complains about a missing argument
696
697 char0 = arg[0]
698
699 # TODO: set - - empty
700 for ch in arg[1:]:
701 #log('ch %r arg_r %s', ch, arg_r)
702 action = spec.actions_short.get(ch)
703 if action is None:
704 e_usage('got invalid flag %r' % ('-' + ch),
705 arg_r.Location())
706
707 attached_arg = char0 if ch in spec.plus_flags else None
708 quit = action.OnMatch(attached_arg, arg_r, out)
709 arg_r.Next() # process the next flag
710
711 if quit:
712 break
713 else:
714 continue
715
716 break # it's a regular arg
717
718 return out