OILS / frontend / args.py View on Github | oils.pub

696 lines, 347 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_t)
58
59from core.error import e_usage
60from mycpp import mops
61from mycpp.mylib import log, iteritems
62
63_ = log
64
65from typing import (Tuple, Optional, Dict, List, TYPE_CHECKING)
66if TYPE_CHECKING:
67 from frontend import flag_spec
68 OptChange = Tuple[str, bool]
69
70# TODO: Move to flag_spec? We use flag_type_t
71String = 1
72Int = 2
73Float = 3 # e.g. for read -t timeout value
74Bool = 4
75
76
77class _Attributes(object):
78 """Object to hold flags.
79
80 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
81 """
82
83 def __init__(self, defaults):
84 # type: (Dict[str, value_t]) -> None
85
86 # New style
87 self.attrs = {} # type: Dict[str, value_t]
88
89 # -o errexit +o nounset
90 self.opt_changes = [] # type: List[OptChange]
91
92 # -O nullglob +O nullglob
93 self.shopt_changes = [] # type: List[OptChange]
94
95 # Special case for --eval and --eval-pure? For bin/osh. It seems OK
96 # to have one now. The pool tells us if it was pure or not.
97 # Note: MAIN_SPEC is different than SET_SPEC, so set --eval does
98 # nothing.
99 self.eval_flags = [] # type: List[Tuple[str, bool]]
100
101 self.show_options = False # 'set -o' without an argument
102 self.actions = [] # type: List[str] # for compgen -A
103 self.saw_double_dash = False # for set --
104 for name, v in iteritems(defaults):
105 self.Set(name, v)
106
107 def SetTrue(self, name):
108 # type: (str) -> None
109 self.Set(name, value.Bool(True))
110
111 def Set(self, name, val):
112 # type: (str, value_t) -> None
113
114 # debug-completion -> debug_completion
115 name = name.replace('-', '_')
116
117 # similar hack to avoid C++ keyword in frontend/flag_gen.py
118 if name == 'extern':
119 name = 'extern_'
120
121 self.attrs[name] = val
122
123 def __repr__(self):
124 # type: () -> str
125 return '<_Attributes %s>' % self.__dict__
126
127
128class Reader(object):
129 """Wrapper for argv.
130
131 Modified by both the parsing loop and various actions.
132
133 The caller of the flags parser can continue to use it after flag parsing is
134 done to get args.
135 """
136
137 def __init__(self, argv, locs=None):
138 # type: (List[str], Optional[List[CompoundWord]]) -> None
139 self.argv = argv
140 self.locs = locs
141 self.n = len(argv)
142 self.i = 0
143
144 def __repr__(self):
145 # type: () -> str
146 return '<args.Reader %r %d>' % (self.argv, self.i)
147
148 def Next(self):
149 # type: () -> None
150 """Advance."""
151 self.i += 1
152
153 def Peek(self):
154 # type: () -> Optional[str]
155 """Return the next token, or None if there are no more.
156
157 None is your SENTINEL for parsing.
158 """
159 if self.i >= self.n:
160 return None
161 else:
162 return self.argv[self.i]
163
164 def Peek2(self):
165 # type: () -> Tuple[Optional[str], loc_t]
166 """Return the next token, or None if there are no more.
167
168 None is your SENTINEL for parsing.
169 """
170 if self.i >= self.n:
171 return None, loc.Missing
172 else:
173 return self.argv[self.i], self.locs[self.i]
174
175 def ReadRequired(self, error_msg):
176 # type: (str) -> str
177 arg = self.Peek()
178 if arg is None:
179 # point at argv[0]
180 e_usage(error_msg, self._FirstLocation())
181 self.Next()
182 return arg
183
184 def ReadRequired2(self, error_msg):
185 # type: (str) -> Tuple[str, CompoundWord]
186 arg = self.Peek()
187 if arg is None:
188 # point at argv[0]
189 e_usage(error_msg, self._FirstLocation())
190 location = self.locs[self.i]
191 self.Next()
192 return arg, location
193
194 def Rest(self):
195 # type: () -> List[str]
196 """Return the rest of the arguments."""
197 return self.argv[self.i:]
198
199 def Rest2(self):
200 # type: () -> Tuple[List[str], List[CompoundWord]]
201 """Return the rest of the arguments."""
202 return self.argv[self.i:], self.locs[self.i:]
203
204 def AtEnd(self):
205 # type: () -> bool
206 return self.i >= self.n # must be >= and not ==
207
208 def Done(self):
209 # type: () -> None
210 if not self.AtEnd():
211 e_usage('got too many arguments', self.Location())
212
213 def _FirstLocation(self):
214 # type: () -> loc_t
215 if self.locs is not None and self.locs[0] is not None:
216 return self.locs[0]
217 else:
218 return loc.Missing
219
220 def Location(self):
221 # type: () -> loc_t
222 if self.locs is not None:
223 if self.i == self.n:
224 i = self.n - 1 # if the last arg is missing, point at the one before
225 else:
226 i = self.i
227 if self.locs[i] is not None:
228 return self.locs[i]
229 else:
230 return loc.Missing
231 else:
232 return loc.Missing
233
234
235class _Action(object):
236 """What is done when a flag or option is detected."""
237
238 def __init__(self):
239 # type: () -> None
240 """Empty constructor for mycpp."""
241 pass
242
243 def OnMatch(self, attached_arg, arg_r, out):
244 # type: (Optional[str], Reader, _Attributes) -> bool
245 """Called when the flag matches.
246
247 Args:
248 prefix: '-' or '+'
249 suffix: ',' for -d,
250 arg_r: Reader() (rename to Input or InputReader?)
251 out: _Attributes() -- the thing we want to set
252
253 Returns:
254 True if flag parsing should be aborted.
255 """
256 raise NotImplementedError()
257
258
259class AppendEvalFlag(_Action):
260
261 def __init__(self, name):
262 # type: (str) -> None
263 _Action.__init__(self)
264 self.name = name
265 self.is_pure = (name == 'eval-pure')
266
267 def OnMatch(self, attached_arg, arg_r, out):
268 # type: (Optional[str], Reader, _Attributes) -> bool
269 """Called when the flag matches."""
270
271 assert attached_arg is None, attached_arg
272
273 arg_r.Next()
274 arg = arg_r.Peek()
275 if arg is None:
276 e_usage('expected argument to %r' % ('--' + self.name),
277 arg_r.Location())
278
279 # is_pure is True for --eval-pure
280 out.eval_flags.append((arg, self.is_pure))
281 return False
282
283
284class _ArgAction(_Action):
285
286 def __init__(self, name, quit_parsing_flags, valid=None):
287 # type: (str, bool, Optional[List[str]]) -> None
288 """
289 Args:
290 quit_parsing_flags: Stop parsing args after this one. for sh -c.
291 python -c behaves the same way.
292 """
293 self.name = name
294 self.quit_parsing_flags = quit_parsing_flags
295 self.valid = valid
296
297 def _Value(self, arg, location):
298 # type: (str, loc_t) -> value_t
299 raise NotImplementedError()
300
301 def OnMatch(self, attached_arg, arg_r, out):
302 # type: (Optional[str], Reader, _Attributes) -> bool
303 """Called when the flag matches."""
304 if attached_arg is not None: # for the ',' in -d,
305 arg = attached_arg
306 else:
307 arg_r.Next()
308 arg = arg_r.Peek()
309 if arg is None:
310 e_usage('expected argument to %r' % ('-' + self.name),
311 arg_r.Location())
312
313 val = self._Value(arg, arg_r.Location())
314 out.Set(self.name, val)
315 return self.quit_parsing_flags
316
317
318class SetToInt(_ArgAction):
319
320 def __init__(self, name):
321 # type: (str) -> None
322 # repeat defaults for C++ translation
323 _ArgAction.__init__(self, name, False, valid=None)
324
325 def _Value(self, arg, location):
326 # type: (str, loc_t) -> value_t
327 #if match.LooksLikeInteger(arg):
328 if True: # break dependency for prebuilt/
329 ok, i = mops.FromStr2(arg)
330 if not ok:
331 #e_usage('Integer too big: %s' % arg, location)
332 e_usage(
333 'expected integer after %s, got %r' %
334 ('-' + self.name, arg), location)
335 else:
336 pass
337 #e_usage(
338 #'expected integer after %s, got %r' % ('-' + self.name, arg),
339 #location)
340
341 # So far all our int values are > 0, so use -1 as the 'unset' value
342 # corner case: this treats -0 as 0!
343 if mops.Greater(mops.BigInt(0), i):
344 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
345 location)
346 return value.Int(i)
347
348
349class SetToFloat(_ArgAction):
350
351 def __init__(self, name):
352 # type: (str) -> None
353 # repeat defaults for C++ translation
354 _ArgAction.__init__(self, name, False, valid=None)
355
356 def _Value(self, arg, location):
357 # type: (str, loc_t) -> value_t
358 try:
359 f = float(arg)
360 except ValueError:
361 e_usage(
362 'expected number after %r, got %r' % ('-' + self.name, arg),
363 location)
364 # So far all our float values are > 0, so use -1.0 as the 'unset' value
365 # corner case: this treats -0.0 as 0.0!
366 if f < 0:
367 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
368 location)
369 return value.Float(f)
370
371
372class SetToString(_ArgAction):
373
374 def __init__(self, name, quit_parsing_flags, valid=None):
375 # type: (str, bool, Optional[List[str]]) -> None
376 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
377
378 def _Value(self, arg, location):
379 # type: (str, loc_t) -> value_t
380 if self.valid is not None and arg not in self.valid:
381 e_usage(
382 'got invalid argument %r to %r, expected one of: %s' %
383 (arg, ('-' + self.name), '|'.join(self.valid)), location)
384 return value.Str(arg)
385
386
387class SetAttachedBool(_Action):
388
389 def __init__(self, name):
390 # type: (str) -> None
391 self.name = name
392
393 def OnMatch(self, attached_arg, arg_r, out):
394 # type: (Optional[str], Reader, _Attributes) -> bool
395 """Called when the flag matches."""
396
397 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
398 #
399 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
400 #
401 # 'attached_arg' is also used for -t0 though, which is weird
402
403 if attached_arg is not None: # '0' in --verbose=0
404 if attached_arg in ('0', 'F', 'false', 'False'):
405 b = False
406 elif attached_arg in ('1', 'T', 'true', 'True'):
407 b = True
408 else:
409 e_usage(
410 'got invalid argument to boolean flag: %r' % attached_arg,
411 loc.Missing)
412 else:
413 b = True
414
415 out.Set(self.name, value.Bool(b))
416 return False
417
418
419class SetToTrue(_Action):
420
421 def __init__(self, name):
422 # type: (str) -> None
423 self.name = name
424
425 def OnMatch(self, attached_arg, arg_r, out):
426 # type: (Optional[str], Reader, _Attributes) -> bool
427 """Called when the flag matches."""
428 out.SetTrue(self.name)
429 return False
430
431
432class SetOption(_Action):
433 """Set an option to a boolean, for 'set +e'."""
434
435 def __init__(self, name):
436 # type: (str) -> None
437 self.name = name
438
439 def OnMatch(self, attached_arg, arg_r, out):
440 # type: (Optional[str], Reader, _Attributes) -> bool
441 """Called when the flag matches."""
442 b = (attached_arg == '-')
443 out.opt_changes.append((self.name, b))
444 return False
445
446
447class SetNamedOption(_Action):
448 """Set a named option to a boolean, for 'set +o errexit'."""
449
450 def __init__(self, shopt=False):
451 # type: (bool) -> None
452 self.names = [] # type: List[str]
453 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
454
455 def ArgName(self, name):
456 # type: (str) -> None
457 self.names.append(name)
458
459 def OnMatch(self, attached_arg, arg_r, out):
460 # type: (Optional[str], Reader, _Attributes) -> bool
461 """Called when the flag matches."""
462 b = (attached_arg == '-')
463 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
464 arg_r.Next() # always advance
465 arg = arg_r.Peek()
466 if arg is None:
467 # triggers on 'set -O' in addition to 'set -o' (meh OK)
468 out.show_options = True
469 return True # quit parsing
470
471 attr_name = arg # Note: validation is done elsewhere
472 if len(self.names) and attr_name not in self.names:
473 e_usage('Invalid option %r' % arg, loc.Missing)
474 changes = out.shopt_changes if self.shopt else out.opt_changes
475 changes.append((attr_name, b))
476 return False
477
478
479class SetAction(_Action):
480 """For compgen -f."""
481
482 def __init__(self, name):
483 # type: (str) -> None
484 self.name = name
485
486 def OnMatch(self, attached_arg, arg_r, out):
487 # type: (Optional[str], Reader, _Attributes) -> bool
488 out.actions.append(self.name)
489 return False
490
491
492class SetNamedAction(_Action):
493 """For compgen -A file."""
494
495 def __init__(self):
496 # type: () -> None
497 self.names = [] # type: List[str]
498
499 def ArgName(self, name):
500 # type: (str) -> None
501 self.names.append(name)
502
503 def OnMatch(self, attached_arg, arg_r, out):
504 # type: (Optional[str], Reader, _Attributes) -> bool
505 """Called when the flag matches."""
506 arg_r.Next() # always advance
507 arg = arg_r.Peek()
508 if arg is None:
509 e_usage('Expected argument for action', loc.Missing)
510
511 attr_name = arg
512 # Validate the option name against a list of valid names.
513 if len(self.names) and attr_name not in self.names:
514 e_usage('Invalid action name %r' % arg, loc.Missing)
515 out.actions.append(attr_name)
516 return False
517
518
519def Parse(spec, arg_r):
520 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
521
522 # NOTE about -:
523 # 'set -' ignores it, vs set
524 # 'unset -' or 'export -' seems to treat it as a variable name
525 out = _Attributes(spec.defaults)
526
527 while not arg_r.AtEnd():
528 arg = arg_r.Peek()
529 if arg == '--':
530 out.saw_double_dash = True
531 arg_r.Next()
532 break
533
534 # Only accept -- if there are any long flags defined
535 if len(spec.actions_long) and arg.startswith('--'):
536 pos = arg.find('=', 2)
537 if pos == -1:
538 suffix = None # type: Optional[str]
539 flag_name = arg[2:] # strip off --
540 else:
541 suffix = arg[pos + 1:]
542 flag_name = arg[2:pos]
543
544 action = spec.actions_long.get(flag_name)
545 if action is None:
546 e_usage('got invalid flag %r' % arg, arg_r.Location())
547
548 action.OnMatch(suffix, arg_r, out)
549 arg_r.Next()
550 continue
551
552 elif arg.startswith('-') and len(arg) > 1:
553 n = len(arg)
554 for i in xrange(1, n): # parse flag combos like -rx
555 ch = arg[i]
556
557 if ch == '0':
558 ch = 'Z' # hack for read -0
559
560 if ch in spec.plus_flags:
561 out.Set(ch, value.Str('-'))
562 continue
563
564 if ch in spec.arity0: # e.g. read -r
565 out.SetTrue(ch)
566 continue
567
568 if ch in spec.arity1: # e.g. read -t1.0
569 action = spec.arity1[ch]
570 # make sure we don't pass empty string for read -t
571 attached_arg = arg[i + 1:] if i < n - 1 else None
572 action.OnMatch(attached_arg, arg_r, out)
573 break
574
575 e_usage("doesn't accept flag %s" % ('-' + ch),
576 arg_r.Location())
577
578 arg_r.Next() # next arg
579
580 # Only accept + if there are ANY options defined, e.g. for declare +rx.
581 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
582 n = len(arg)
583 for i in xrange(1, n): # parse flag combos like -rx
584 ch = arg[i]
585 if ch in spec.plus_flags:
586 out.Set(ch, value.Str('+'))
587 continue
588
589 e_usage("doesn't accept option %s" % ('+' + ch),
590 arg_r.Location())
591
592 arg_r.Next() # next arg
593
594 else: # a regular arg
595 break
596
597 return out
598
599
600def ParseLikeEcho(spec, arg_r):
601 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
602 """Echo is a special case. These work: echo -n echo -en.
603
604 - But don't respect --
605 - doesn't fail when an invalid flag is passed
606 """
607 out = _Attributes(spec.defaults)
608
609 while not arg_r.AtEnd():
610 arg = arg_r.Peek()
611 chars = arg[1:]
612 if arg.startswith('-') and len(chars):
613 # Check if it looks like -en or not. TODO: could optimize this.
614 done = False
615 for c in chars:
616 if c not in spec.arity0:
617 done = True
618 break
619 if done:
620 break
621
622 for ch in chars:
623 out.SetTrue(ch)
624
625 else:
626 break # Looks like an arg
627
628 arg_r.Next() # next arg
629
630 return out
631
632
633def ParseMore(spec, arg_r):
634 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
635 """Return attributes and an index.
636
637 Respects +, like set +eu
638
639 We do NOT respect:
640
641 WRONG: sh -cecho OK: sh -c echo
642 WRONG: set -opipefail OK: set -o pipefail
643
644 But we do accept these
645
646 set -euo pipefail
647 set -oeu pipefail
648 set -oo pipefail errexit
649 """
650 out = _Attributes(spec.defaults)
651
652 quit = False
653 while not arg_r.AtEnd():
654 arg = arg_r.Peek()
655 if arg == '--':
656 out.saw_double_dash = True
657 arg_r.Next()
658 break
659
660 if arg.startswith('--'):
661 action = spec.actions_long.get(arg[2:])
662 if action is None:
663 e_usage('got invalid flag %r' % arg, arg_r.Location())
664
665 # Note: not parsing --foo=bar as attached_arg, as above
666 action.OnMatch(None, arg_r, out)
667 arg_r.Next()
668 continue
669
670 # corner case: sh +c is also accepted!
671 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
672 # note: we're not handling sh -cecho (no space) as an argument
673 # It complains about a missing argument
674
675 char0 = arg[0]
676
677 # TODO: set - - empty
678 for ch in arg[1:]:
679 #log('ch %r arg_r %s', ch, arg_r)
680 action = spec.actions_short.get(ch)
681 if action is None:
682 e_usage('got invalid flag %r' % ('-' + ch),
683 arg_r.Location())
684
685 attached_arg = char0 if ch in spec.plus_flags else None
686 quit = action.OnMatch(attached_arg, arg_r, out)
687 arg_r.Next() # process the next flag
688
689 if quit:
690 break
691 else:
692 continue
693
694 break # it's a regular arg
695
696 return out