OILS / frontend / args.py View on Github | oils.pub

712 lines, 362 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, iteritems
62
63_ = log
64
65from typing import (Tuple, Optional, Dict, List, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 # -o errexit +o nounset
90 self.opt_changes = [] # type: List[OptChange]
91
92 # -O nullglob +O nullglob
93 self.shopt_changes = [] # type: List[OptChange]
94
95 # Special case for --eval and --eval-pure? For bin/osh. It seems OK
96 # to have one now. The pool tells us if it was pure or not.
97 # Note: MAIN_SPEC is different than SET_SPEC, so set --eval does
98 # nothing.
99 self.eval_flags = [] # type: List[Tuple[str, bool]]
100
101 self.show_options = False # 'set -o' without an argument
102 self.actions = [] # type: List[str] # for compgen -A
103 self.saw_double_dash = False # for set --
104 self.saw_single_dash = False # for set -
105 for name, v in iteritems(defaults):
106 self.Set(name, v)
107
108 def SetTrue(self, name):
109 # type: (str) -> None
110 self.Set(name, value.Bool(True))
111
112 def Set(self, name, val):
113 # type: (str, value_t) -> None
114
115 # debug-completion -> debug_completion
116 name = name.replace('-', '_')
117
118 # similar hack to avoid C++ keyword in frontend/flag_gen.py
119 if name == 'extern':
120 name = 'extern_'
121 elif name == 'private':
122 name = 'private_'
123
124 self.attrs[name] = val
125
126 def __repr__(self):
127 # type: () -> str
128 return '<_Attributes %s>' % self.__dict__
129
130
131class Reader(object):
132 """Wrapper for argv.
133
134 Modified by both the parsing loop and various actions.
135
136 The caller of the flags parser can continue to use it after flag parsing is
137 done to get args.
138 """
139
140 def __init__(self, argv, locs=None):
141 # type: (List[str], Optional[List[CompoundWord]]) -> None
142 self.argv = argv
143 self.locs = locs
144 self.n = len(argv)
145 self.i = 0
146
147 def __repr__(self):
148 # type: () -> str
149 return '<args.Reader %r %d>' % (self.argv, self.i)
150
151 def Next(self):
152 # type: () -> None
153 """Advance."""
154 self.i += 1
155
156 def Peek(self):
157 # type: () -> Optional[str]
158 """Return the next token, or None if there are no more.
159
160 None is your SENTINEL for parsing.
161 """
162 if self.i >= self.n:
163 return None
164 else:
165 return self.argv[self.i]
166
167 def Peek2(self):
168 # type: () -> Tuple[Optional[str], loc_t]
169 """Return the next token, or None if there are no more.
170
171 None is your SENTINEL for parsing.
172 """
173 if self.i >= self.n:
174 return None, loc.Missing
175 else:
176 return self.argv[self.i], self.locs[self.i]
177
178 def ReadRequired(self, error_msg):
179 # type: (str) -> str
180 arg = self.Peek()
181 if arg is None:
182 # point at argv[0]
183 e_usage(error_msg, self._FirstLocation())
184 self.Next()
185 return arg
186
187 def ReadRequired2(self, error_msg):
188 # type: (str) -> Tuple[str, CompoundWord]
189 arg = self.Peek()
190 if arg is None:
191 # point at argv[0]
192 e_usage(error_msg, self._FirstLocation())
193 location = self.locs[self.i]
194 self.Next()
195 return arg, location
196
197 def Rest(self):
198 # type: () -> List[str]
199 """Return the rest of the arguments."""
200 return self.argv[self.i:]
201
202 def Rest2(self):
203 # type: () -> Tuple[List[str], List[CompoundWord]]
204 """Return the rest of the arguments."""
205 return self.argv[self.i:], self.locs[self.i:]
206
207 def AtEnd(self):
208 # type: () -> bool
209 return self.i >= self.n # must be >= and not ==
210
211 def Done(self):
212 # type: () -> None
213 if not self.AtEnd():
214 e_usage('got too many arguments', self.Location())
215
216 def _FirstLocation(self):
217 # type: () -> loc_t
218 if self.locs is not None and self.locs[0] is not None:
219 return self.locs[0]
220 else:
221 return loc.Missing
222
223 def Location(self):
224 # type: () -> loc_t
225 if self.locs is not None:
226 if self.i == self.n:
227 i = self.n - 1 # if the last arg is missing, point at the one before
228 else:
229 i = self.i
230 if self.locs[i] is not None:
231 return self.locs[i]
232 else:
233 return loc.Missing
234 else:
235 return loc.Missing
236
237
238class _Action(object):
239 """What is done when a flag or option is detected."""
240
241 def __init__(self):
242 # type: () -> None
243 """Empty constructor for mycpp."""
244 pass
245
246 def OnMatch(self, attached_arg, arg_r, out):
247 # type: (Optional[str], Reader, _Attributes) -> bool
248 """Called when the flag matches.
249
250 Args:
251 prefix: '-' or '+'
252 suffix: ',' for -d,
253 arg_r: Reader() (rename to Input or InputReader?)
254 out: _Attributes() -- the thing we want to set
255
256 Returns:
257 True if flag parsing should be aborted.
258 """
259 raise NotImplementedError()
260
261
262class AppendEvalFlag(_Action):
263
264 def __init__(self, name):
265 # type: (str) -> None
266 _Action.__init__(self)
267 self.name = name
268 self.is_pure = (name == 'eval-pure')
269
270 def OnMatch(self, attached_arg, arg_r, out):
271 # type: (Optional[str], Reader, _Attributes) -> bool
272 """Called when the flag matches."""
273
274 assert attached_arg is None, attached_arg
275
276 arg_r.Next()
277 arg = arg_r.Peek()
278 if arg is None:
279 e_usage('expected argument to %r' % ('--' + self.name),
280 arg_r.Location())
281
282 # is_pure is True for --eval-pure
283 out.eval_flags.append((arg, self.is_pure))
284 return False
285
286
287class _ArgAction(_Action):
288
289 def __init__(self, name, valid=None):
290 # type: (str, Optional[List[str]]) -> None
291 """
292 Args:
293 """
294 self.name = name
295 self.valid = valid
296
297 def _Value(self, arg, location):
298 # type: (str, loc_t) -> value_t
299 raise NotImplementedError()
300
301 def OnMatch(self, attached_arg, arg_r, out):
302 # type: (Optional[str], Reader, _Attributes) -> bool
303 """Called when the flag matches."""
304 if attached_arg is not None: # for the ',' in -d,
305 arg = attached_arg
306 else:
307 arg_r.Next()
308 arg = arg_r.Peek()
309 if arg is None:
310 e_usage("expected argument to '-%s'" % self.name,
311 arg_r.Location())
312
313 val = self._Value(arg, arg_r.Location())
314 out.Set(self.name, val)
315 return False
316
317
318class SetToInt(_ArgAction):
319
320 def __init__(self, name):
321 # type: (str) -> None
322 # repeat defaults for C++ translation
323 _ArgAction.__init__(self, name, valid=None)
324
325 def _Value(self, arg, location):
326 # type: (str, loc_t) -> value_t
327 #if match.LooksLikeInteger(arg):
328 if True: # break dependency for prebuilt/
329 ok, i = mops.FromStr2(arg)
330 if not ok:
331 #e_usage('Integer too big: %s' % arg, location)
332 e_usage(
333 'expected integer after %s, got %r' %
334 ('-' + self.name, arg), location)
335 else:
336 pass
337 #e_usage(
338 #'expected integer after %s, got %r' % ('-' + self.name, arg),
339 #location)
340
341 # So far all our int values are > 0, so use -1 as the 'unset' value
342 # corner case: this treats -0 as 0!
343 if mops.Greater(mops.BigInt(0), i):
344 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
345 location)
346 return value.Int(i)
347
348
349class SetToFloat(_ArgAction):
350
351 def __init__(self, name):
352 # type: (str) -> None
353 # repeat defaults for C++ translation
354 _ArgAction.__init__(self, name, valid=None)
355
356 def _Value(self, arg, location):
357 # type: (str, loc_t) -> value_t
358 try:
359 f = float(arg)
360 except ValueError:
361 e_usage(
362 'expected number after %r, got %r' % ('-' + self.name, arg),
363 location)
364 # So far all our float values are > 0, so use -1.0 as the 'unset' value
365 # corner case: this treats -0.0 as 0.0!
366 if f < 0:
367 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
368 location)
369 return value.Float(f)
370
371
372class SetToString(_ArgAction):
373
374 def __init__(self, name, valid=None):
375 # type: (str, Optional[List[str]]) -> None
376 _ArgAction.__init__(self, name, valid=valid)
377
378 def _Value(self, arg, location):
379 # type: (str, loc_t) -> value_t
380 if self.valid is not None and arg not in self.valid:
381 e_usage(
382 'got invalid argument %r to %r, expected one of: %s' %
383 (arg, ('-' + self.name), '|'.join(self.valid)), location)
384 return value.Str(arg)
385
386
387class SetAttachedBool(_Action):
388
389 def __init__(self, name):
390 # type: (str) -> None
391 self.name = name
392
393 def OnMatch(self, attached_arg, arg_r, out):
394 # type: (Optional[str], Reader, _Attributes) -> bool
395 """Called when the flag matches."""
396
397 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
398 #
399 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
400 #
401 # 'attached_arg' is also used for -t0 though, which is weird
402
403 if attached_arg is not None: # '0' in --verbose=0
404 if attached_arg in ('0', 'F', 'false', 'False'):
405 b = False
406 elif attached_arg in ('1', 'T', 'true', 'True'):
407 b = True
408 else:
409 e_usage(
410 'got invalid argument to boolean flag: %r' % attached_arg,
411 loc.Missing)
412 else:
413 b = True
414
415 out.Set(self.name, value.Bool(b))
416 return False
417
418
419class SetToTrue(_Action):
420
421 def __init__(self, name):
422 # type: (str) -> None
423 self.name = name
424
425 def OnMatch(self, attached_arg, arg_r, out):
426 # type: (Optional[str], Reader, _Attributes) -> bool
427 """Called when the flag matches."""
428 out.SetTrue(self.name)
429 return False
430
431
432class SetOption(_Action):
433 """Set an option to a boolean, for 'set +e'."""
434
435 def __init__(self, name):
436 # type: (str) -> None
437 self.name = name
438
439 def OnMatch(self, attached_arg, arg_r, out):
440 # type: (Optional[str], Reader, _Attributes) -> bool
441 """Called when the flag matches."""
442 b = (attached_arg == '-')
443 out.opt_changes.append((self.name, b))
444 return False
445
446
447class SetNamedOption(_Action):
448 """Set a named option to a boolean, for 'set +o errexit'."""
449
450 def __init__(self, shopt=False):
451 # type: (bool) -> None
452 self.names = [] # type: List[str]
453 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
454
455 def ArgName(self, name):
456 # type: (str) -> None
457 self.names.append(name)
458
459 def OnMatch(self, attached_arg, arg_r, out):
460 # type: (Optional[str], Reader, _Attributes) -> bool
461 """Called when the flag matches."""
462 b = (attached_arg == '-')
463 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
464 arg_r.Next() # always advance
465 arg = arg_r.Peek()
466 if arg is None:
467 # triggers on 'set -O' in addition to 'set -o' (meh OK)
468 out.show_options = True
469 return True # quit parsing
470
471 attr_name = arg # Note: validation is done elsewhere
472 if len(self.names) and attr_name not in self.names:
473 e_usage('Invalid option %r' % arg, loc.Missing)
474 changes = out.shopt_changes if self.shopt else out.opt_changes
475 changes.append((attr_name, b))
476 return False
477
478
479class SetAction(_Action):
480 """For compgen -f."""
481
482 def __init__(self, name):
483 # type: (str) -> None
484 self.name = name
485
486 def OnMatch(self, attached_arg, arg_r, out):
487 # type: (Optional[str], Reader, _Attributes) -> bool
488 out.actions.append(self.name)
489 return False
490
491
492class SetNamedAction(_Action):
493 """For compgen -A file."""
494
495 def __init__(self):
496 # type: () -> None
497 self.names = [] # type: List[str]
498
499 def ArgName(self, name):
500 # type: (str) -> None
501 self.names.append(name)
502
503 def OnMatch(self, attached_arg, arg_r, out):
504 # type: (Optional[str], Reader, _Attributes) -> bool
505 """Called when the flag matches."""
506 arg_r.Next() # always advance
507 arg = arg_r.Peek()
508 if arg is None:
509 e_usage('Expected argument for action', loc.Missing)
510
511 attr_name = arg
512 # Validate the option name against a list of valid names.
513 if len(self.names) and attr_name not in self.names:
514 e_usage('Invalid action name %r' % arg, loc.Missing)
515 out.actions.append(attr_name)
516 return False
517
518
519def Parse(spec, arg_r):
520 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
521
522 # NOTE about -:
523 # 'set -' ignores it, vs set
524 # 'unset -' or 'export -' seems to treat it as a variable name
525 out = _Attributes(spec.defaults)
526
527 while not arg_r.AtEnd():
528 arg = arg_r.Peek()
529 if arg == '--':
530 out.saw_double_dash = True
531 arg_r.Next()
532 break
533
534 # Only accept -- if there are any long flags defined
535 if len(spec.actions_long) and arg.startswith('--'):
536 pos = arg.find('=', 2)
537 if pos == -1:
538 suffix = None # type: Optional[str]
539 flag_name = arg[2:] # strip off --
540 else:
541 suffix = arg[pos + 1:]
542 flag_name = arg[2:pos]
543
544 action = spec.actions_long.get(flag_name)
545 if action is None:
546 e_usage('got invalid flag %r' % arg, arg_r.Location())
547
548 action.OnMatch(suffix, arg_r, out)
549 arg_r.Next()
550 continue
551
552 elif arg.startswith('-') and len(arg) > 1:
553 n = len(arg)
554 for i in xrange(1, n): # parse flag combos like -rx
555 ch = arg[i]
556
557 if ch == '0':
558 ch = 'Z' # hack for read -0
559
560 if ch in spec.plus_flags:
561 out.Set(ch, value.Str('-'))
562 continue
563
564 if ch in spec.arity0: # e.g. read -r
565 out.SetTrue(ch)
566 continue
567
568 if ch in spec.arity1: # e.g. read -t1.0
569 action = spec.arity1[ch]
570 # make sure we don't pass empty string for read -t
571 attached_arg = arg[i + 1:] if i < n - 1 else None
572 action.OnMatch(attached_arg, arg_r, out)
573 break
574
575 e_usage("doesn't accept flag %s" % ('-' + ch),
576 arg_r.Location())
577
578 arg_r.Next() # next arg
579
580 # Only accept + if there are ANY options defined, e.g. for declare +rx.
581 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
582 n = len(arg)
583 for i in xrange(1, n): # parse flag combos like -rx
584 ch = arg[i]
585 if ch in spec.plus_flags:
586 out.Set(ch, value.Str('+'))
587 continue
588
589 e_usage("doesn't accept option %s" % ('+' + ch),
590 arg_r.Location())
591
592 arg_r.Next() # next arg
593
594 else: # a regular arg
595 break
596
597 return out
598
599
600def ParseLikeEcho(spec, arg_r):
601 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
602 """Echo is a special case. These work: echo -n echo -en.
603
604 - But don't respect --
605 - doesn't fail when an invalid flag is passed
606 """
607 out = _Attributes(spec.defaults)
608
609 while not arg_r.AtEnd():
610 arg = arg_r.Peek()
611 chars = arg[1:]
612 if arg.startswith('-') and len(chars):
613 # Check if it looks like -en or not. TODO: could optimize this.
614 done = False
615 for c in chars:
616 if c not in spec.arity0:
617 done = True
618 break
619 if done:
620 break
621
622 for ch in chars:
623 out.SetTrue(ch)
624
625 else:
626 break # Looks like an arg
627
628 arg_r.Next() # next arg
629
630 return out
631
632
633def ParseMore(spec, arg_r, sh_dash_c=False):
634 # type: (flag_spec._FlagSpecAndMore, Reader, bool) -> _Attributes
635 """Return attributes and an index.
636
637 Respects +, like set +eu
638
639 We do NOT respect:
640
641 WRONG: sh -cecho OK: sh -c echo
642 WRONG: set -opipefail OK: set -o pipefail
643
644 But we do accept these
645
646 set -euo pipefail
647 set -oeu pipefail
648 set -oo pipefail errexit
649 """
650 out = _Attributes(spec.defaults)
651
652 set_dash_c = False
653 while not arg_r.AtEnd():
654 arg = arg_r.Peek()
655 if arg == '--':
656 out.saw_double_dash = True
657 arg_r.Next()
658 break
659
660 if arg == '-': # weird special behavior for 'set -'
661 out.saw_single_dash = True
662 arg_r.Next()
663 break
664
665 if arg == '+': # a single + is an ignored flag
666 arg_r.Next()
667 continue
668
669 if arg.startswith('--'):
670 action = spec.actions_long.get(arg[2:])
671 if action is None:
672 e_usage('got invalid flag %r' % arg, arg_r.Location())
673
674 # Note: not parsing --foo=bar as attached_arg, as above
675 action.OnMatch(None, arg_r, out)
676 arg_r.Next()
677 continue
678
679 # corner case: sh +c is also accepted!
680 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
681 # note: we're not handling sh -cecho (no space) as an argument
682 # It complains about a missing argument
683
684 char0 = arg[0] # - or +
685
686 for ch in arg[1:]: # -xyz foo is like -x -y foo -z
687
688 if sh_dash_c and ch == 'c': # special case
689 set_dash_c = True # handle below
690 continue
691
692 action = spec.actions_short.get(ch)
693 if action is None:
694 e_usage('got invalid flag %r' % ('-' + ch),
695 arg_r.Location())
696
697 attached_arg = char0 if ch in spec.plus_flags else None
698 action.OnMatch(attached_arg, arg_r, out)
699
700 arg_r.Next() # process the next flag
701 continue
702
703 break # it's a regular arg
704
705 if set_dash_c:
706 cmd = arg_r.Peek()
707 if cmd is None:
708 e_usage("expected argument to '-c'", loc.Missing)
709 out.Set('c', value.Str(cmd))
710 arg_r.Next()
711
712 return out