OILS / frontend / args.py View on Github | oilshell.org

681 lines, 348 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60#from frontend import match
61from mycpp import mops
62from mycpp.mylib import log, tagswitch, iteritems
63
64_ = log
65
66from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
67if TYPE_CHECKING:
68 from frontend import flag_spec
69 OptChange = Tuple[str, bool]
70
71# TODO: Move to flag_spec? We use flag_type_t
72String = 1
73Int = 2
74Float = 3 # e.g. for read -t timeout value
75Bool = 4
76
77
78class _Attributes(object):
79 """Object to hold flags.
80
81 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
82 """
83
84 def __init__(self, defaults):
85 # type: (Dict[str, value_t]) -> None
86
87 # New style
88 self.attrs = {} # type: Dict[str, value_t]
89
90 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
91 self.shopt_changes = [
92 ] # type: List[OptChange] # -O nullglob +O nullglob
93 self.show_options = False # 'set -o' without an argument
94 self.actions = [] # type: List[str] # for compgen -A
95 self.saw_double_dash = False # for set --
96 for name, v in iteritems(defaults):
97 self.Set(name, v)
98
99 def SetTrue(self, name):
100 # type: (str) -> None
101 self.Set(name, value.Bool(True))
102
103 def Set(self, name, val):
104 # type: (str, value_t) -> None
105
106 # debug-completion -> debug_completion
107 name = name.replace('-', '_')
108
109 # similar hack to avoid C++ keyword in frontend/flag_gen.py
110 if name == 'extern':
111 name = 'extern_'
112
113 self.attrs[name] = val
114
115 if 0:
116 # Backward compatibility!
117 with tagswitch(val) as case:
118 if case(value_e.Undef):
119 py_val = None # type: Any
120 elif case(value_e.Bool):
121 py_val = cast(value.Bool, val).b
122 elif case(value_e.Int):
123 py_val = cast(value.Int, val).i
124 elif case(value_e.Float):
125 py_val = cast(value.Float, val).f
126 elif case(value_e.Str):
127 py_val = cast(value.Str, val).s
128 else:
129 raise AssertionError(val)
130
131 setattr(self, name, py_val)
132
133 def __repr__(self):
134 # type: () -> str
135 return '<_Attributes %s>' % self.__dict__
136
137
138class Reader(object):
139 """Wrapper for argv.
140
141 Modified by both the parsing loop and various actions.
142
143 The caller of the flags parser can continue to use it after flag parsing is
144 done to get args.
145 """
146
147 def __init__(self, argv, locs=None):
148 # type: (List[str], Optional[List[CompoundWord]]) -> None
149 self.argv = argv
150 self.locs = locs
151 self.n = len(argv)
152 self.i = 0
153
154 def __repr__(self):
155 # type: () -> str
156 return '<args.Reader %r %d>' % (self.argv, self.i)
157
158 def Next(self):
159 # type: () -> None
160 """Advance."""
161 self.i += 1
162
163 def Peek(self):
164 # type: () -> Optional[str]
165 """Return the next token, or None if there are no more.
166
167 None is your SENTINEL for parsing.
168 """
169 if self.i >= self.n:
170 return None
171 else:
172 return self.argv[self.i]
173
174 def Peek2(self):
175 # type: () -> Tuple[Optional[str], loc_t]
176 """Return the next token, or None if there are no more.
177
178 None is your SENTINEL for parsing.
179 """
180 if self.i >= self.n:
181 return None, loc.Missing
182 else:
183 return self.argv[self.i], self.locs[self.i]
184
185 def ReadRequired(self, error_msg):
186 # type: (str) -> str
187 arg = self.Peek()
188 if arg is None:
189 # point at argv[0]
190 e_usage(error_msg, self._FirstLocation())
191 self.Next()
192 return arg
193
194 def ReadRequired2(self, error_msg):
195 # type: (str) -> Tuple[str, loc_t]
196 arg = self.Peek()
197 if arg is None:
198 # point at argv[0]
199 e_usage(error_msg, self._FirstLocation())
200 location = self.locs[self.i]
201 self.Next()
202 return arg, location
203
204 def Rest(self):
205 # type: () -> List[str]
206 """Return the rest of the arguments."""
207 return self.argv[self.i:]
208
209 def Rest2(self):
210 # type: () -> Tuple[List[str], List[CompoundWord]]
211 """Return the rest of the arguments."""
212 return self.argv[self.i:], self.locs[self.i:]
213
214 def AtEnd(self):
215 # type: () -> bool
216 return self.i >= self.n # must be >= and not ==
217
218 def Done(self):
219 # type: () -> None
220 if not self.AtEnd():
221 e_usage('got too many arguments', self.Location())
222
223 def _FirstLocation(self):
224 # type: () -> loc_t
225 if self.locs is not None and self.locs[0] is not None:
226 return self.locs[0]
227 else:
228 return loc.Missing
229
230 def Location(self):
231 # type: () -> loc_t
232 if self.locs is not None:
233 if self.i == self.n:
234 i = self.n - 1 # if the last arg is missing, point at the one before
235 else:
236 i = self.i
237 if self.locs[i] is not None:
238 return self.locs[i]
239 else:
240 return loc.Missing
241 else:
242 return loc.Missing
243
244
245class _Action(object):
246 """What is done when a flag or option is detected."""
247
248 def __init__(self):
249 # type: () -> None
250 """Empty constructor for mycpp."""
251 pass
252
253 def OnMatch(self, attached_arg, arg_r, out):
254 # type: (Optional[str], Reader, _Attributes) -> bool
255 """Called when the flag matches.
256
257 Args:
258 prefix: '-' or '+'
259 suffix: ',' for -d,
260 arg_r: Reader() (rename to Input or InputReader?)
261 out: _Attributes() -- the thing we want to set
262
263 Returns:
264 True if flag parsing should be aborted.
265 """
266 raise NotImplementedError()
267
268
269class _ArgAction(_Action):
270
271 def __init__(self, name, quit_parsing_flags, valid=None):
272 # type: (str, bool, Optional[List[str]]) -> None
273 """
274 Args:
275 quit_parsing_flags: Stop parsing args after this one. for sh -c.
276 python -c behaves the same way.
277 """
278 self.name = name
279 self.quit_parsing_flags = quit_parsing_flags
280 self.valid = valid
281
282 def _Value(self, arg, location):
283 # type: (str, loc_t) -> value_t
284 raise NotImplementedError()
285
286 def OnMatch(self, attached_arg, arg_r, out):
287 # type: (Optional[str], Reader, _Attributes) -> bool
288 """Called when the flag matches."""
289 if attached_arg is not None: # for the ',' in -d,
290 arg = attached_arg
291 else:
292 arg_r.Next()
293 arg = arg_r.Peek()
294 if arg is None:
295 e_usage('expected argument to %r' % ('-' + self.name),
296 arg_r.Location())
297
298 val = self._Value(arg, arg_r.Location())
299 out.Set(self.name, val)
300 return self.quit_parsing_flags
301
302
303class SetToInt(_ArgAction):
304
305 def __init__(self, name):
306 # type: (str) -> None
307 # repeat defaults for C++ translation
308 _ArgAction.__init__(self, name, False, valid=None)
309
310 def _Value(self, arg, location):
311 # type: (str, loc_t) -> value_t
312 #if match.LooksLikeInteger(arg):
313 if True: # break dependency for prebuilt/
314 ok, i = mops.FromStr2(arg)
315 if not ok:
316 #e_usage('Integer too big: %s' % arg, location)
317 e_usage(
318 'expected integer after %s, got %r' %
319 ('-' + self.name, arg), location)
320 else:
321 pass
322 #e_usage(
323 #'expected integer after %s, got %r' % ('-' + self.name, arg),
324 #location)
325
326 # So far all our int values are > 0, so use -1 as the 'unset' value
327 # corner case: this treats -0 as 0!
328 if mops.Greater(mops.BigInt(0), i):
329 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
330 location)
331 return value.Int(i)
332
333
334class SetToFloat(_ArgAction):
335
336 def __init__(self, name):
337 # type: (str) -> None
338 # repeat defaults for C++ translation
339 _ArgAction.__init__(self, name, False, valid=None)
340
341 def _Value(self, arg, location):
342 # type: (str, loc_t) -> value_t
343 try:
344 f = float(arg)
345 except ValueError:
346 e_usage(
347 'expected number after %r, got %r' % ('-' + self.name, arg),
348 location)
349 # So far all our float values are > 0, so use -1.0 as the 'unset' value
350 # corner case: this treats -0.0 as 0.0!
351 if f < 0:
352 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
353 location)
354 return value.Float(f)
355
356
357class SetToString(_ArgAction):
358
359 def __init__(self, name, quit_parsing_flags, valid=None):
360 # type: (str, bool, Optional[List[str]]) -> None
361 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
362
363 def _Value(self, arg, location):
364 # type: (str, loc_t) -> value_t
365 if self.valid is not None and arg not in self.valid:
366 e_usage(
367 'got invalid argument %r to %r, expected one of: %s' %
368 (arg, ('-' + self.name), '|'.join(self.valid)), location)
369 return value.Str(arg)
370
371
372class SetAttachedBool(_Action):
373
374 def __init__(self, name):
375 # type: (str) -> None
376 self.name = name
377
378 def OnMatch(self, attached_arg, arg_r, out):
379 # type: (Optional[str], Reader, _Attributes) -> bool
380 """Called when the flag matches."""
381
382 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
383 #
384 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
385 #
386 # 'attached_arg' is also used for -t0 though, which is weird
387
388 if attached_arg is not None: # '0' in --verbose=0
389 if attached_arg in ('0', 'F', 'false', 'False'):
390 b = False
391 elif attached_arg in ('1', 'T', 'true', 'True'):
392 b = True
393 else:
394 e_usage(
395 'got invalid argument to boolean flag: %r' % attached_arg,
396 loc.Missing)
397 else:
398 b = True
399
400 out.Set(self.name, value.Bool(b))
401 return False
402
403
404class SetToTrue(_Action):
405
406 def __init__(self, name):
407 # type: (str) -> None
408 self.name = name
409
410 def OnMatch(self, attached_arg, arg_r, out):
411 # type: (Optional[str], Reader, _Attributes) -> bool
412 """Called when the flag matches."""
413 out.SetTrue(self.name)
414 return False
415
416
417class SetOption(_Action):
418 """Set an option to a boolean, for 'set +e'."""
419
420 def __init__(self, name):
421 # type: (str) -> None
422 self.name = name
423
424 def OnMatch(self, attached_arg, arg_r, out):
425 # type: (Optional[str], Reader, _Attributes) -> bool
426 """Called when the flag matches."""
427 b = (attached_arg == '-')
428 out.opt_changes.append((self.name, b))
429 return False
430
431
432class SetNamedOption(_Action):
433 """Set a named option to a boolean, for 'set +o errexit'."""
434
435 def __init__(self, shopt=False):
436 # type: (bool) -> None
437 self.names = [] # type: List[str]
438 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
439
440 def ArgName(self, name):
441 # type: (str) -> None
442 self.names.append(name)
443
444 def OnMatch(self, attached_arg, arg_r, out):
445 # type: (Optional[str], Reader, _Attributes) -> bool
446 """Called when the flag matches."""
447 b = (attached_arg == '-')
448 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
449 arg_r.Next() # always advance
450 arg = arg_r.Peek()
451 if arg is None:
452 # triggers on 'set -O' in addition to 'set -o' (meh OK)
453 out.show_options = True
454 return True # quit parsing
455
456 attr_name = arg # Note: validation is done elsewhere
457 if len(self.names) and attr_name not in self.names:
458 e_usage('Invalid option %r' % arg, loc.Missing)
459 changes = out.shopt_changes if self.shopt else out.opt_changes
460 changes.append((attr_name, b))
461 return False
462
463
464class SetAction(_Action):
465 """For compgen -f."""
466
467 def __init__(self, name):
468 # type: (str) -> None
469 self.name = name
470
471 def OnMatch(self, attached_arg, arg_r, out):
472 # type: (Optional[str], Reader, _Attributes) -> bool
473 out.actions.append(self.name)
474 return False
475
476
477class SetNamedAction(_Action):
478 """For compgen -A file."""
479
480 def __init__(self):
481 # type: () -> None
482 self.names = [] # type: List[str]
483
484 def ArgName(self, name):
485 # type: (str) -> None
486 self.names.append(name)
487
488 def OnMatch(self, attached_arg, arg_r, out):
489 # type: (Optional[str], Reader, _Attributes) -> bool
490 """Called when the flag matches."""
491 arg_r.Next() # always advance
492 arg = arg_r.Peek()
493 if arg is None:
494 e_usage('Expected argument for action', loc.Missing)
495
496 attr_name = arg
497 # Validate the option name against a list of valid names.
498 if len(self.names) and attr_name not in self.names:
499 e_usage('Invalid action name %r' % arg, loc.Missing)
500 out.actions.append(attr_name)
501 return False
502
503
504def Parse(spec, arg_r):
505 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
506
507 # NOTE about -:
508 # 'set -' ignores it, vs set
509 # 'unset -' or 'export -' seems to treat it as a variable name
510 out = _Attributes(spec.defaults)
511
512 while not arg_r.AtEnd():
513 arg = arg_r.Peek()
514 if arg == '--':
515 out.saw_double_dash = True
516 arg_r.Next()
517 break
518
519 # Only accept -- if there are any long flags defined
520 if len(spec.actions_long) and arg.startswith('--'):
521 pos = arg.find('=', 2)
522 if pos == -1:
523 suffix = None # type: Optional[str]
524 flag_name = arg[2:] # strip off --
525 else:
526 suffix = arg[pos + 1:]
527 flag_name = arg[2:pos]
528
529 action = spec.actions_long.get(flag_name)
530 if action is None:
531 e_usage('got invalid flag %r' % arg, arg_r.Location())
532
533 action.OnMatch(suffix, arg_r, out)
534 arg_r.Next()
535 continue
536
537 elif arg.startswith('-') and len(arg) > 1:
538 n = len(arg)
539 for i in xrange(1, n): # parse flag combos like -rx
540 ch = arg[i]
541
542 if ch == '0':
543 ch = 'Z' # hack for read -0
544
545 if ch in spec.plus_flags:
546 out.Set(ch, value.Str('-'))
547 continue
548
549 if ch in spec.arity0: # e.g. read -r
550 out.SetTrue(ch)
551 continue
552
553 if ch in spec.arity1: # e.g. read -t1.0
554 action = spec.arity1[ch]
555 # make sure we don't pass empty string for read -t
556 attached_arg = arg[i + 1:] if i < n - 1 else None
557 action.OnMatch(attached_arg, arg_r, out)
558 break
559
560 e_usage("doesn't accept flag %s" % ('-' + ch),
561 arg_r.Location())
562
563 arg_r.Next() # next arg
564
565 # Only accept + if there are ANY options defined, e.g. for declare +rx.
566 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
567 n = len(arg)
568 for i in xrange(1, n): # parse flag combos like -rx
569 ch = arg[i]
570 if ch in spec.plus_flags:
571 out.Set(ch, value.Str('+'))
572 continue
573
574 e_usage("doesn't accept option %s" % ('+' + ch),
575 arg_r.Location())
576
577 arg_r.Next() # next arg
578
579 else: # a regular arg
580 break
581
582 return out
583
584
585def ParseLikeEcho(spec, arg_r):
586 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
587 """Echo is a special case. These work: echo -n echo -en.
588
589 - But don't respect --
590 - doesn't fail when an invalid flag is passed
591 """
592 out = _Attributes(spec.defaults)
593
594 while not arg_r.AtEnd():
595 arg = arg_r.Peek()
596 chars = arg[1:]
597 if arg.startswith('-') and len(chars):
598 # Check if it looks like -en or not. TODO: could optimize this.
599 done = False
600 for c in chars:
601 if c not in spec.arity0:
602 done = True
603 break
604 if done:
605 break
606
607 for ch in chars:
608 out.SetTrue(ch)
609
610 else:
611 break # Looks like an arg
612
613 arg_r.Next() # next arg
614
615 return out
616
617
618def ParseMore(spec, arg_r):
619 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
620 """Return attributes and an index.
621
622 Respects +, like set +eu
623
624 We do NOT respect:
625
626 WRONG: sh -cecho OK: sh -c echo
627 WRONG: set -opipefail OK: set -o pipefail
628
629 But we do accept these
630
631 set -euo pipefail
632 set -oeu pipefail
633 set -oo pipefail errexit
634 """
635 out = _Attributes(spec.defaults)
636
637 quit = False
638 while not arg_r.AtEnd():
639 arg = arg_r.Peek()
640 if arg == '--':
641 out.saw_double_dash = True
642 arg_r.Next()
643 break
644
645 if arg.startswith('--'):
646 action = spec.actions_long.get(arg[2:])
647 if action is None:
648 e_usage('got invalid flag %r' % arg, arg_r.Location())
649
650 # Note: not parsing --foo=bar as attached_arg, as above
651 action.OnMatch(None, arg_r, out)
652 arg_r.Next()
653 continue
654
655 # corner case: sh +c is also accepted!
656 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
657 # note: we're not handling sh -cecho (no space) as an argument
658 # It complains about a missing argument
659
660 char0 = arg[0]
661
662 # TODO: set - - empty
663 for ch in arg[1:]:
664 #log('ch %r arg_r %s', ch, arg_r)
665 action = spec.actions_short.get(ch)
666 if action is None:
667 e_usage('got invalid flag %r' % ('-' + ch),
668 arg_r.Location())
669
670 attached_arg = char0 if ch in spec.plus_flags else None
671 quit = action.OnMatch(attached_arg, arg_r, out)
672 arg_r.Next() # process the next flag
673
674 if quit:
675 break
676 else:
677 continue
678
679 break # it's a regular arg
680
681 return out