OILS / frontend / args.py View on Github | oils.pub

708 lines, 357 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, iteritems
62
63_ = log
64
65from typing import (Tuple, Optional, Dict, List, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 # -o errexit +o nounset
90 self.opt_changes = [] # type: List[OptChange]
91
92 # -O nullglob +O nullglob
93 self.shopt_changes = [] # type: List[OptChange]
94
95 # Special case for --eval and --eval-pure? For bin/osh. It seems OK
96 # to have one now. The pool tells us if it was pure or not.
97 # Note: MAIN_SPEC is different than SET_SPEC, so set --eval does
98 # nothing.
99 self.eval_flags = [] # type: List[Tuple[str, bool]]
100
101 self.show_options = False # 'set -o' without an argument
102 self.actions = [] # type: List[str] # for compgen -A
103 self.saw_double_dash = False # for set --
104 self.saw_single_dash = False # for set -
105 for name, v in iteritems(defaults):
106 self.Set(name, v)
107
108 def SetTrue(self, name):
109 # type: (str) -> None
110 self.Set(name, value.Bool(True))
111
112 def Set(self, name, val):
113 # type: (str, value_t) -> None
114
115 # debug-completion -> debug_completion
116 name = name.replace('-', '_')
117
118 # similar hack to avoid C++ keyword in frontend/flag_gen.py
119 if name == 'extern':
120 name = 'extern_'
121 elif name == 'private':
122 name = 'private_'
123
124 self.attrs[name] = val
125
126 def __repr__(self):
127 # type: () -> str
128 return '<_Attributes %s>' % self.__dict__
129
130
131class Reader(object):
132 """Wrapper for argv.
133
134 Modified by both the parsing loop and various actions.
135
136 The caller of the flags parser can continue to use it after flag parsing is
137 done to get args.
138 """
139
140 def __init__(self, argv, locs=None):
141 # type: (List[str], Optional[List[CompoundWord]]) -> None
142 self.argv = argv
143 self.locs = locs
144 self.n = len(argv)
145 self.i = 0
146
147 def __repr__(self):
148 # type: () -> str
149 return '<args.Reader %r %d>' % (self.argv, self.i)
150
151 def Next(self):
152 # type: () -> None
153 """Advance."""
154 self.i += 1
155
156 def Peek(self):
157 # type: () -> Optional[str]
158 """Return the next token, or None if there are no more.
159
160 None is your SENTINEL for parsing.
161 """
162 if self.i >= self.n:
163 return None
164 else:
165 return self.argv[self.i]
166
167 def Peek2(self):
168 # type: () -> Tuple[Optional[str], loc_t]
169 """Return the next token, or None if there are no more.
170
171 None is your SENTINEL for parsing.
172 """
173 if self.i >= self.n:
174 return None, loc.Missing
175 else:
176 return self.argv[self.i], self.locs[self.i]
177
178 def ReadRequired(self, error_msg):
179 # type: (str) -> str
180 arg = self.Peek()
181 if arg is None:
182 # point at argv[0]
183 e_usage(error_msg, self._FirstLocation())
184 self.Next()
185 return arg
186
187 def ReadRequired2(self, error_msg):
188 # type: (str) -> Tuple[str, CompoundWord]
189 arg = self.Peek()
190 if arg is None:
191 # point at argv[0]
192 e_usage(error_msg, self._FirstLocation())
193 location = self.locs[self.i]
194 self.Next()
195 return arg, location
196
197 def Rest(self):
198 # type: () -> List[str]
199 """Return the rest of the arguments."""
200 return self.argv[self.i:]
201
202 def Rest2(self):
203 # type: () -> Tuple[List[str], List[CompoundWord]]
204 """Return the rest of the arguments."""
205 return self.argv[self.i:], self.locs[self.i:]
206
207 def AtEnd(self):
208 # type: () -> bool
209 return self.i >= self.n # must be >= and not ==
210
211 def Done(self):
212 # type: () -> None
213 if not self.AtEnd():
214 e_usage('got too many arguments', self.Location())
215
216 def _FirstLocation(self):
217 # type: () -> loc_t
218 if self.locs is not None and self.locs[0] is not None:
219 return self.locs[0]
220 else:
221 return loc.Missing
222
223 def Location(self):
224 # type: () -> loc_t
225 if self.locs is not None:
226 if self.i == self.n:
227 i = self.n - 1 # if the last arg is missing, point at the one before
228 else:
229 i = self.i
230 if self.locs[i] is not None:
231 return self.locs[i]
232 else:
233 return loc.Missing
234 else:
235 return loc.Missing
236
237
238class _Action(object):
239 """What is done when a flag or option is detected."""
240
241 def __init__(self):
242 # type: () -> None
243 """Empty constructor for mycpp."""
244 pass
245
246 def OnMatch(self, attached_arg, arg_r, out):
247 # type: (Optional[str], Reader, _Attributes) -> bool
248 """Called when the flag matches.
249
250 Args:
251 prefix: '-' or '+'
252 suffix: ',' for -d,
253 arg_r: Reader() (rename to Input or InputReader?)
254 out: _Attributes() -- the thing we want to set
255
256 Returns:
257 True if flag parsing should be aborted.
258 """
259 raise NotImplementedError()
260
261
262class AppendEvalFlag(_Action):
263
264 def __init__(self, name):
265 # type: (str) -> None
266 _Action.__init__(self)
267 self.name = name
268 self.is_pure = (name == 'eval-pure')
269
270 def OnMatch(self, attached_arg, arg_r, out):
271 # type: (Optional[str], Reader, _Attributes) -> bool
272 """Called when the flag matches."""
273
274 assert attached_arg is None, attached_arg
275
276 arg_r.Next()
277 arg = arg_r.Peek()
278 if arg is None:
279 e_usage('expected argument to %r' % ('--' + self.name),
280 arg_r.Location())
281
282 # is_pure is True for --eval-pure
283 out.eval_flags.append((arg, self.is_pure))
284 return False
285
286
287class _ArgAction(_Action):
288
289 def __init__(self, name, quit_parsing_flags, valid=None):
290 # type: (str, bool, Optional[List[str]]) -> None
291 """
292 Args:
293 quit_parsing_flags: Stop parsing args after this one. for sh -c.
294 python -c behaves the same way.
295 """
296 self.name = name
297 self.quit_parsing_flags = quit_parsing_flags
298 self.valid = valid
299
300 def _Value(self, arg, location):
301 # type: (str, loc_t) -> value_t
302 raise NotImplementedError()
303
304 def OnMatch(self, attached_arg, arg_r, out):
305 # type: (Optional[str], Reader, _Attributes) -> bool
306 """Called when the flag matches."""
307 if attached_arg is not None: # for the ',' in -d,
308 arg = attached_arg
309 else:
310 arg_r.Next()
311 arg = arg_r.Peek()
312 if arg is None:
313 e_usage('expected argument to %r' % ('-' + self.name),
314 arg_r.Location())
315
316 val = self._Value(arg, arg_r.Location())
317 out.Set(self.name, val)
318 return self.quit_parsing_flags
319
320
321class SetToInt(_ArgAction):
322
323 def __init__(self, name):
324 # type: (str) -> None
325 # repeat defaults for C++ translation
326 _ArgAction.__init__(self, name, False, valid=None)
327
328 def _Value(self, arg, location):
329 # type: (str, loc_t) -> value_t
330 #if match.LooksLikeInteger(arg):
331 if True: # break dependency for prebuilt/
332 ok, i = mops.FromStr2(arg)
333 if not ok:
334 #e_usage('Integer too big: %s' % arg, location)
335 e_usage(
336 'expected integer after %s, got %r' %
337 ('-' + self.name, arg), location)
338 else:
339 pass
340 #e_usage(
341 #'expected integer after %s, got %r' % ('-' + self.name, arg),
342 #location)
343
344 # So far all our int values are > 0, so use -1 as the 'unset' value
345 # corner case: this treats -0 as 0!
346 if mops.Greater(mops.BigInt(0), i):
347 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
348 location)
349 return value.Int(i)
350
351
352class SetToFloat(_ArgAction):
353
354 def __init__(self, name):
355 # type: (str) -> None
356 # repeat defaults for C++ translation
357 _ArgAction.__init__(self, name, False, valid=None)
358
359 def _Value(self, arg, location):
360 # type: (str, loc_t) -> value_t
361 try:
362 f = float(arg)
363 except ValueError:
364 e_usage(
365 'expected number after %r, got %r' % ('-' + self.name, arg),
366 location)
367 # So far all our float values are > 0, so use -1.0 as the 'unset' value
368 # corner case: this treats -0.0 as 0.0!
369 if f < 0:
370 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
371 location)
372 return value.Float(f)
373
374
375class SetToString(_ArgAction):
376
377 def __init__(self, name, quit_parsing_flags, valid=None):
378 # type: (str, bool, Optional[List[str]]) -> None
379 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
380
381 def _Value(self, arg, location):
382 # type: (str, loc_t) -> value_t
383 if self.valid is not None and arg not in self.valid:
384 e_usage(
385 'got invalid argument %r to %r, expected one of: %s' %
386 (arg, ('-' + self.name), '|'.join(self.valid)), location)
387 return value.Str(arg)
388
389
390class SetAttachedBool(_Action):
391
392 def __init__(self, name):
393 # type: (str) -> None
394 self.name = name
395
396 def OnMatch(self, attached_arg, arg_r, out):
397 # type: (Optional[str], Reader, _Attributes) -> bool
398 """Called when the flag matches."""
399
400 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
401 #
402 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
403 #
404 # 'attached_arg' is also used for -t0 though, which is weird
405
406 if attached_arg is not None: # '0' in --verbose=0
407 if attached_arg in ('0', 'F', 'false', 'False'):
408 b = False
409 elif attached_arg in ('1', 'T', 'true', 'True'):
410 b = True
411 else:
412 e_usage(
413 'got invalid argument to boolean flag: %r' % attached_arg,
414 loc.Missing)
415 else:
416 b = True
417
418 out.Set(self.name, value.Bool(b))
419 return False
420
421
422class SetToTrue(_Action):
423
424 def __init__(self, name):
425 # type: (str) -> None
426 self.name = name
427
428 def OnMatch(self, attached_arg, arg_r, out):
429 # type: (Optional[str], Reader, _Attributes) -> bool
430 """Called when the flag matches."""
431 out.SetTrue(self.name)
432 return False
433
434
435class SetOption(_Action):
436 """Set an option to a boolean, for 'set +e'."""
437
438 def __init__(self, name):
439 # type: (str) -> None
440 self.name = name
441
442 def OnMatch(self, attached_arg, arg_r, out):
443 # type: (Optional[str], Reader, _Attributes) -> bool
444 """Called when the flag matches."""
445 b = (attached_arg == '-')
446 out.opt_changes.append((self.name, b))
447 return False
448
449
450class SetNamedOption(_Action):
451 """Set a named option to a boolean, for 'set +o errexit'."""
452
453 def __init__(self, shopt=False):
454 # type: (bool) -> None
455 self.names = [] # type: List[str]
456 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
457
458 def ArgName(self, name):
459 # type: (str) -> None
460 self.names.append(name)
461
462 def OnMatch(self, attached_arg, arg_r, out):
463 # type: (Optional[str], Reader, _Attributes) -> bool
464 """Called when the flag matches."""
465 b = (attached_arg == '-')
466 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
467 arg_r.Next() # always advance
468 arg = arg_r.Peek()
469 if arg is None:
470 # triggers on 'set -O' in addition to 'set -o' (meh OK)
471 out.show_options = True
472 return True # quit parsing
473
474 attr_name = arg # Note: validation is done elsewhere
475 if len(self.names) and attr_name not in self.names:
476 e_usage('Invalid option %r' % arg, loc.Missing)
477 changes = out.shopt_changes if self.shopt else out.opt_changes
478 changes.append((attr_name, b))
479 return False
480
481
482class SetAction(_Action):
483 """For compgen -f."""
484
485 def __init__(self, name):
486 # type: (str) -> None
487 self.name = name
488
489 def OnMatch(self, attached_arg, arg_r, out):
490 # type: (Optional[str], Reader, _Attributes) -> bool
491 out.actions.append(self.name)
492 return False
493
494
495class SetNamedAction(_Action):
496 """For compgen -A file."""
497
498 def __init__(self):
499 # type: () -> None
500 self.names = [] # type: List[str]
501
502 def ArgName(self, name):
503 # type: (str) -> None
504 self.names.append(name)
505
506 def OnMatch(self, attached_arg, arg_r, out):
507 # type: (Optional[str], Reader, _Attributes) -> bool
508 """Called when the flag matches."""
509 arg_r.Next() # always advance
510 arg = arg_r.Peek()
511 if arg is None:
512 e_usage('Expected argument for action', loc.Missing)
513
514 attr_name = arg
515 # Validate the option name against a list of valid names.
516 if len(self.names) and attr_name not in self.names:
517 e_usage('Invalid action name %r' % arg, loc.Missing)
518 out.actions.append(attr_name)
519 return False
520
521
522def Parse(spec, arg_r):
523 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
524
525 # NOTE about -:
526 # 'set -' ignores it, vs set
527 # 'unset -' or 'export -' seems to treat it as a variable name
528 out = _Attributes(spec.defaults)
529
530 while not arg_r.AtEnd():
531 arg = arg_r.Peek()
532 if arg == '--':
533 out.saw_double_dash = True
534 arg_r.Next()
535 break
536
537 # Only accept -- if there are any long flags defined
538 if len(spec.actions_long) and arg.startswith('--'):
539 pos = arg.find('=', 2)
540 if pos == -1:
541 suffix = None # type: Optional[str]
542 flag_name = arg[2:] # strip off --
543 else:
544 suffix = arg[pos + 1:]
545 flag_name = arg[2:pos]
546
547 action = spec.actions_long.get(flag_name)
548 if action is None:
549 e_usage('got invalid flag %r' % arg, arg_r.Location())
550
551 action.OnMatch(suffix, arg_r, out)
552 arg_r.Next()
553 continue
554
555 elif arg.startswith('-') and len(arg) > 1:
556 n = len(arg)
557 for i in xrange(1, n): # parse flag combos like -rx
558 ch = arg[i]
559
560 if ch == '0':
561 ch = 'Z' # hack for read -0
562
563 if ch in spec.plus_flags:
564 out.Set(ch, value.Str('-'))
565 continue
566
567 if ch in spec.arity0: # e.g. read -r
568 out.SetTrue(ch)
569 continue
570
571 if ch in spec.arity1: # e.g. read -t1.0
572 action = spec.arity1[ch]
573 # make sure we don't pass empty string for read -t
574 attached_arg = arg[i + 1:] if i < n - 1 else None
575 action.OnMatch(attached_arg, arg_r, out)
576 break
577
578 e_usage("doesn't accept flag %s" % ('-' + ch),
579 arg_r.Location())
580
581 arg_r.Next() # next arg
582
583 # Only accept + if there are ANY options defined, e.g. for declare +rx.
584 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
585 n = len(arg)
586 for i in xrange(1, n): # parse flag combos like -rx
587 ch = arg[i]
588 if ch in spec.plus_flags:
589 out.Set(ch, value.Str('+'))
590 continue
591
592 e_usage("doesn't accept option %s" % ('+' + ch),
593 arg_r.Location())
594
595 arg_r.Next() # next arg
596
597 else: # a regular arg
598 break
599
600 return out
601
602
603def ParseLikeEcho(spec, arg_r):
604 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
605 """Echo is a special case. These work: echo -n echo -en.
606
607 - But don't respect --
608 - doesn't fail when an invalid flag is passed
609 """
610 out = _Attributes(spec.defaults)
611
612 while not arg_r.AtEnd():
613 arg = arg_r.Peek()
614 chars = arg[1:]
615 if arg.startswith('-') and len(chars):
616 # Check if it looks like -en or not. TODO: could optimize this.
617 done = False
618 for c in chars:
619 if c not in spec.arity0:
620 done = True
621 break
622 if done:
623 break
624
625 for ch in chars:
626 out.SetTrue(ch)
627
628 else:
629 break # Looks like an arg
630
631 arg_r.Next() # next arg
632
633 return out
634
635
636def ParseMore(spec, arg_r):
637 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
638 """Return attributes and an index.
639
640 Respects +, like set +eu
641
642 We do NOT respect:
643
644 WRONG: sh -cecho OK: sh -c echo
645 WRONG: set -opipefail OK: set -o pipefail
646
647 But we do accept these
648
649 set -euo pipefail
650 set -oeu pipefail
651 set -oo pipefail errexit
652 """
653 out = _Attributes(spec.defaults)
654
655 quit = False
656 while not arg_r.AtEnd():
657 arg = arg_r.Peek()
658 if arg == '--':
659 out.saw_double_dash = True
660 arg_r.Next()
661 break
662
663 if arg == '-': # weird special behavior for 'set -'
664 out.saw_single_dash = True
665 arg_r.Next()
666 break
667
668 if arg == '+': # a single + is an ignored flag
669 arg_r.Next()
670 continue
671
672 if arg.startswith('--'):
673 action = spec.actions_long.get(arg[2:])
674 if action is None:
675 e_usage('got invalid flag %r' % arg, arg_r.Location())
676
677 # Note: not parsing --foo=bar as attached_arg, as above
678 action.OnMatch(None, arg_r, out)
679 arg_r.Next()
680 continue
681
682 # corner case: sh +c is also accepted!
683 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
684 # note: we're not handling sh -cecho (no space) as an argument
685 # It complains about a missing argument
686
687 char0 = arg[0]
688
689 # TODO: set - - empty
690 for ch in arg[1:]:
691 #log('ch %r arg_r %s', ch, arg_r)
692 action = spec.actions_short.get(ch)
693 if action is None:
694 e_usage('got invalid flag %r' % ('-' + ch),
695 arg_r.Location())
696
697 attached_arg = char0 if ch in spec.plus_flags else None
698 quit = action.OnMatch(attached_arg, arg_r, out)
699 arg_r.Next() # process the next flag
700
701 if quit:
702 break
703 else:
704 continue
705
706 break # it's a regular arg
707
708 return out