OILS / frontend / args.py View on Github | oilshell.org

670 lines, 346 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from frontend import match
61from mycpp import mops
62from mycpp.mylib import log, tagswitch, iteritems
63
64_ = log
65
66from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
67if TYPE_CHECKING:
68 from frontend import flag_spec
69 OptChange = Tuple[str, bool]
70
71# TODO: Move to flag_spec? We use flag_type_t
72String = 1
73Int = 2
74Float = 3 # e.g. for read -t timeout value
75Bool = 4
76
77
78class _Attributes(object):
79 """Object to hold flags.
80
81 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
82 """
83
84 def __init__(self, defaults):
85 # type: (Dict[str, value_t]) -> None
86
87 # New style
88 self.attrs = {} # type: Dict[str, value_t]
89
90 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
91 self.shopt_changes = [
92 ] # type: List[OptChange] # -O nullglob +O nullglob
93 self.show_options = False # 'set -o' without an argument
94 self.actions = [] # type: List[str] # for compgen -A
95 self.saw_double_dash = False # for set --
96 for name, v in iteritems(defaults):
97 self.Set(name, v)
98
99 def SetTrue(self, name):
100 # type: (str) -> None
101 self.Set(name, value.Bool(True))
102
103 def Set(self, name, val):
104 # type: (str, value_t) -> None
105
106 # debug-completion -> debug_completion
107 name = name.replace('-', '_')
108 self.attrs[name] = val
109
110 if 0:
111 # Backward compatibility!
112 with tagswitch(val) as case:
113 if case(value_e.Undef):
114 py_val = None # type: Any
115 elif case(value_e.Bool):
116 py_val = cast(value.Bool, val).b
117 elif case(value_e.Int):
118 py_val = cast(value.Int, val).i
119 elif case(value_e.Float):
120 py_val = cast(value.Float, val).f
121 elif case(value_e.Str):
122 py_val = cast(value.Str, val).s
123 else:
124 raise AssertionError(val)
125
126 setattr(self, name, py_val)
127
128 def __repr__(self):
129 # type: () -> str
130 return '<_Attributes %s>' % self.__dict__
131
132
133class Reader(object):
134 """Wrapper for argv.
135
136 Modified by both the parsing loop and various actions.
137
138 The caller of the flags parser can continue to use it after flag parsing is
139 done to get args.
140 """
141
142 def __init__(self, argv, locs=None):
143 # type: (List[str], Optional[List[CompoundWord]]) -> None
144 self.argv = argv
145 self.locs = locs
146 self.n = len(argv)
147 self.i = 0
148
149 def __repr__(self):
150 # type: () -> str
151 return '<args.Reader %r %d>' % (self.argv, self.i)
152
153 def Next(self):
154 # type: () -> None
155 """Advance."""
156 self.i += 1
157
158 def Peek(self):
159 # type: () -> Optional[str]
160 """Return the next token, or None if there are no more.
161
162 None is your SENTINEL for parsing.
163 """
164 if self.i >= self.n:
165 return None
166 else:
167 return self.argv[self.i]
168
169 def Peek2(self):
170 # type: () -> Tuple[Optional[str], loc_t]
171 """Return the next token, or None if there are no more.
172
173 None is your SENTINEL for parsing.
174 """
175 if self.i >= self.n:
176 return None, loc.Missing
177 else:
178 return self.argv[self.i], self.locs[self.i]
179
180 def ReadRequired(self, error_msg):
181 # type: (str) -> str
182 arg = self.Peek()
183 if arg is None:
184 # point at argv[0]
185 e_usage(error_msg, self._FirstLocation())
186 self.Next()
187 return arg
188
189 def ReadRequired2(self, error_msg):
190 # type: (str) -> Tuple[str, loc_t]
191 arg = self.Peek()
192 if arg is None:
193 # point at argv[0]
194 e_usage(error_msg, self._FirstLocation())
195 location = self.locs[self.i]
196 self.Next()
197 return arg, location
198
199 def Rest(self):
200 # type: () -> List[str]
201 """Return the rest of the arguments."""
202 return self.argv[self.i:]
203
204 def Rest2(self):
205 # type: () -> Tuple[List[str], List[CompoundWord]]
206 """Return the rest of the arguments."""
207 return self.argv[self.i:], self.locs[self.i:]
208
209 def AtEnd(self):
210 # type: () -> bool
211 return self.i >= self.n # must be >= and not ==
212
213 def Done(self):
214 # type: () -> None
215 if not self.AtEnd():
216 e_usage('got too many arguments', self.Location())
217
218 def _FirstLocation(self):
219 # type: () -> loc_t
220 if self.locs is not None and self.locs[0] is not None:
221 return self.locs[0]
222 else:
223 return loc.Missing
224
225 def Location(self):
226 # type: () -> loc_t
227 if self.locs is not None:
228 if self.i == self.n:
229 i = self.n - 1 # if the last arg is missing, point at the one before
230 else:
231 i = self.i
232 if self.locs[i] is not None:
233 return self.locs[i]
234 else:
235 return loc.Missing
236 else:
237 return loc.Missing
238
239
240class _Action(object):
241 """What is done when a flag or option is detected."""
242
243 def __init__(self):
244 # type: () -> None
245 """Empty constructor for mycpp."""
246 pass
247
248 def OnMatch(self, attached_arg, arg_r, out):
249 # type: (Optional[str], Reader, _Attributes) -> bool
250 """Called when the flag matches.
251
252 Args:
253 prefix: '-' or '+'
254 suffix: ',' for -d,
255 arg_r: Reader() (rename to Input or InputReader?)
256 out: _Attributes() -- the thing we want to set
257
258 Returns:
259 True if flag parsing should be aborted.
260 """
261 raise NotImplementedError()
262
263
264class _ArgAction(_Action):
265
266 def __init__(self, name, quit_parsing_flags, valid=None):
267 # type: (str, bool, Optional[List[str]]) -> None
268 """
269 Args:
270 quit_parsing_flags: Stop parsing args after this one. for sh -c.
271 python -c behaves the same way.
272 """
273 self.name = name
274 self.quit_parsing_flags = quit_parsing_flags
275 self.valid = valid
276
277 def _Value(self, arg, location):
278 # type: (str, loc_t) -> value_t
279 raise NotImplementedError()
280
281 def OnMatch(self, attached_arg, arg_r, out):
282 # type: (Optional[str], Reader, _Attributes) -> bool
283 """Called when the flag matches."""
284 if attached_arg is not None: # for the ',' in -d,
285 arg = attached_arg
286 else:
287 arg_r.Next()
288 arg = arg_r.Peek()
289 if arg is None:
290 e_usage('expected argument to %r' % ('-' + self.name),
291 arg_r.Location())
292
293 val = self._Value(arg, arg_r.Location())
294 out.Set(self.name, val)
295 return self.quit_parsing_flags
296
297
298class SetToInt(_ArgAction):
299
300 def __init__(self, name):
301 # type: (str) -> None
302 # repeat defaults for C++ translation
303 _ArgAction.__init__(self, name, False, valid=None)
304
305 def _Value(self, arg, location):
306 # type: (str, loc_t) -> value_t
307 if match.LooksLikeInteger(arg):
308 i = mops.FromStr(arg)
309 else:
310 e_usage(
311 'expected integer after %s, got %r' % ('-' + self.name, arg),
312 location)
313
314 # So far all our int values are > 0, so use -1 as the 'unset' value
315 # corner case: this treats -0 as 0!
316 if mops.Greater(mops.BigInt(0), i):
317 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
318 location)
319 return value.Int(i)
320
321
322class SetToFloat(_ArgAction):
323
324 def __init__(self, name):
325 # type: (str) -> None
326 # repeat defaults for C++ translation
327 _ArgAction.__init__(self, name, False, valid=None)
328
329 def _Value(self, arg, location):
330 # type: (str, loc_t) -> value_t
331 try:
332 f = float(arg)
333 except ValueError:
334 e_usage(
335 'expected number after %r, got %r' % ('-' + self.name, arg),
336 location)
337 # So far all our float values are > 0, so use -1.0 as the 'unset' value
338 # corner case: this treats -0.0 as 0.0!
339 if f < 0:
340 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
341 location)
342 return value.Float(f)
343
344
345class SetToString(_ArgAction):
346
347 def __init__(self, name, quit_parsing_flags, valid=None):
348 # type: (str, bool, Optional[List[str]]) -> None
349 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
350
351 def _Value(self, arg, location):
352 # type: (str, loc_t) -> value_t
353 if self.valid is not None and arg not in self.valid:
354 e_usage(
355 'got invalid argument %r to %r, expected one of: %s' %
356 (arg, ('-' + self.name), '|'.join(self.valid)), location)
357 return value.Str(arg)
358
359
360class SetAttachedBool(_Action):
361
362 def __init__(self, name):
363 # type: (str) -> None
364 self.name = name
365
366 def OnMatch(self, attached_arg, arg_r, out):
367 # type: (Optional[str], Reader, _Attributes) -> bool
368 """Called when the flag matches."""
369
370 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
371 #
372 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
373 #
374 # 'attached_arg' is also used for -t0 though, which is weird
375
376 if attached_arg is not None: # '0' in --verbose=0
377 if attached_arg in ('0', 'F', 'false',
378 'False'): # TODO: incorrect translation
379 b = False
380 elif attached_arg in ('1', 'T', 'true', 'Talse'):
381 b = True
382 else:
383 e_usage(
384 'got invalid argument to boolean flag: %r' % attached_arg,
385 loc.Missing)
386 else:
387 b = True
388
389 out.Set(self.name, value.Bool(b))
390 return False
391
392
393class SetToTrue(_Action):
394
395 def __init__(self, name):
396 # type: (str) -> None
397 self.name = name
398
399 def OnMatch(self, attached_arg, arg_r, out):
400 # type: (Optional[str], Reader, _Attributes) -> bool
401 """Called when the flag matches."""
402 out.SetTrue(self.name)
403 return False
404
405
406class SetOption(_Action):
407 """Set an option to a boolean, for 'set +e'."""
408
409 def __init__(self, name):
410 # type: (str) -> None
411 self.name = name
412
413 def OnMatch(self, attached_arg, arg_r, out):
414 # type: (Optional[str], Reader, _Attributes) -> bool
415 """Called when the flag matches."""
416 b = (attached_arg == '-')
417 out.opt_changes.append((self.name, b))
418 return False
419
420
421class SetNamedOption(_Action):
422 """Set a named option to a boolean, for 'set +o errexit'."""
423
424 def __init__(self, shopt=False):
425 # type: (bool) -> None
426 self.names = [] # type: List[str]
427 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
428
429 def ArgName(self, name):
430 # type: (str) -> None
431 self.names.append(name)
432
433 def OnMatch(self, attached_arg, arg_r, out):
434 # type: (Optional[str], Reader, _Attributes) -> bool
435 """Called when the flag matches."""
436 b = (attached_arg == '-')
437 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
438 arg_r.Next() # always advance
439 arg = arg_r.Peek()
440 if arg is None:
441 # triggers on 'set -O' in addition to 'set -o' (meh OK)
442 out.show_options = True
443 return True # quit parsing
444
445 attr_name = arg # Note: validation is done elsewhere
446 if len(self.names) and attr_name not in self.names:
447 e_usage('Invalid option %r' % arg, loc.Missing)
448 changes = out.shopt_changes if self.shopt else out.opt_changes
449 changes.append((attr_name, b))
450 return False
451
452
453class SetAction(_Action):
454 """For compgen -f."""
455
456 def __init__(self, name):
457 # type: (str) -> None
458 self.name = name
459
460 def OnMatch(self, attached_arg, arg_r, out):
461 # type: (Optional[str], Reader, _Attributes) -> bool
462 out.actions.append(self.name)
463 return False
464
465
466class SetNamedAction(_Action):
467 """For compgen -A file."""
468
469 def __init__(self):
470 # type: () -> None
471 self.names = [] # type: List[str]
472
473 def ArgName(self, name):
474 # type: (str) -> None
475 self.names.append(name)
476
477 def OnMatch(self, attached_arg, arg_r, out):
478 # type: (Optional[str], Reader, _Attributes) -> bool
479 """Called when the flag matches."""
480 arg_r.Next() # always advance
481 arg = arg_r.Peek()
482 if arg is None:
483 e_usage('Expected argument for action', loc.Missing)
484
485 attr_name = arg
486 # Validate the option name against a list of valid names.
487 if len(self.names) and attr_name not in self.names:
488 e_usage('Invalid action name %r' % arg, loc.Missing)
489 out.actions.append(attr_name)
490 return False
491
492
493def Parse(spec, arg_r):
494 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
495
496 # NOTE about -:
497 # 'set -' ignores it, vs set
498 # 'unset -' or 'export -' seems to treat it as a variable name
499 out = _Attributes(spec.defaults)
500
501 while not arg_r.AtEnd():
502 arg = arg_r.Peek()
503 if arg == '--':
504 out.saw_double_dash = True
505 arg_r.Next()
506 break
507
508 # Only accept -- if there are any long flags defined
509 if len(spec.actions_long) and arg.startswith('--'):
510 pos = arg.find('=', 2)
511 if pos == -1:
512 suffix = None # type: Optional[str]
513 flag_name = arg[2:] # strip off --
514 else:
515 suffix = arg[pos + 1:]
516 flag_name = arg[2:pos]
517
518 action = spec.actions_long.get(flag_name)
519 if action is None:
520 e_usage('got invalid flag %r' % arg, arg_r.Location())
521
522 action.OnMatch(suffix, arg_r, out)
523 arg_r.Next()
524 continue
525
526 elif arg.startswith('-') and len(arg) > 1:
527 n = len(arg)
528 for i in xrange(1, n): # parse flag combos like -rx
529 ch = arg[i]
530
531 if ch == '0':
532 ch = 'Z' # hack for read -0
533
534 if ch in spec.plus_flags:
535 out.Set(ch, value.Str('-'))
536 continue
537
538 if ch in spec.arity0: # e.g. read -r
539 out.SetTrue(ch)
540 continue
541
542 if ch in spec.arity1: # e.g. read -t1.0
543 action = spec.arity1[ch]
544 # make sure we don't pass empty string for read -t
545 attached_arg = arg[i + 1:] if i < n - 1 else None
546 action.OnMatch(attached_arg, arg_r, out)
547 break
548
549 e_usage("doesn't accept flag %s" % ('-' + ch),
550 arg_r.Location())
551
552 arg_r.Next() # next arg
553
554 # Only accept + if there are ANY options defined, e.g. for declare +rx.
555 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
556 n = len(arg)
557 for i in xrange(1, n): # parse flag combos like -rx
558 ch = arg[i]
559 if ch in spec.plus_flags:
560 out.Set(ch, value.Str('+'))
561 continue
562
563 e_usage("doesn't accept option %s" % ('+' + ch),
564 arg_r.Location())
565
566 arg_r.Next() # next arg
567
568 else: # a regular arg
569 break
570
571 return out
572
573
574def ParseLikeEcho(spec, arg_r):
575 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
576 """Echo is a special case. These work: echo -n echo -en.
577
578 - But don't respect --
579 - doesn't fail when an invalid flag is passed
580 """
581 out = _Attributes(spec.defaults)
582
583 while not arg_r.AtEnd():
584 arg = arg_r.Peek()
585 chars = arg[1:]
586 if arg.startswith('-') and len(chars):
587 # Check if it looks like -en or not. TODO: could optimize this.
588 done = False
589 for c in chars:
590 if c not in spec.arity0:
591 done = True
592 break
593 if done:
594 break
595
596 for ch in chars:
597 out.SetTrue(ch)
598
599 else:
600 break # Looks like an arg
601
602 arg_r.Next() # next arg
603
604 return out
605
606
607def ParseMore(spec, arg_r):
608 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
609 """Return attributes and an index.
610
611 Respects +, like set +eu
612
613 We do NOT respect:
614
615 WRONG: sh -cecho OK: sh -c echo
616 WRONG: set -opipefail OK: set -o pipefail
617
618 But we do accept these
619
620 set -euo pipefail
621 set -oeu pipefail
622 set -oo pipefail errexit
623 """
624 out = _Attributes(spec.defaults)
625
626 quit = False
627 while not arg_r.AtEnd():
628 arg = arg_r.Peek()
629 if arg == '--':
630 out.saw_double_dash = True
631 arg_r.Next()
632 break
633
634 if arg.startswith('--'):
635 action = spec.actions_long.get(arg[2:])
636 if action is None:
637 e_usage('got invalid flag %r' % arg, arg_r.Location())
638
639 # Note: not parsing --foo=bar as attached_arg, as above
640 action.OnMatch(None, arg_r, out)
641 arg_r.Next()
642 continue
643
644 # corner case: sh +c is also accepted!
645 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
646 # note: we're not handling sh -cecho (no space) as an argument
647 # It complains about a missing argument
648
649 char0 = arg[0]
650
651 # TODO: set - - empty
652 for ch in arg[1:]:
653 #log('ch %r arg_r %s', ch, arg_r)
654 action = spec.actions_short.get(ch)
655 if action is None:
656 e_usage('got invalid flag %r' % ('-' + ch),
657 arg_r.Location())
658
659 attached_arg = char0 if ch in spec.plus_flags else None
660 quit = action.OnMatch(attached_arg, arg_r, out)
661 arg_r.Next() # process the next flag
662
663 if quit:
664 break
665 else:
666 continue
667
668 break # it's a regular arg
669
670 return out