OILS / frontend / args.py View on Github | oilshell.org

682 lines, 349 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60#from frontend import match
61from mycpp import mops
62from mycpp.mylib import log, tagswitch, iteritems
63
64_ = log
65
66from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
67if TYPE_CHECKING:
68 from frontend import flag_spec
69 OptChange = Tuple[str, bool]
70
71# TODO: Move to flag_spec? We use flag_type_t
72String = 1
73Int = 2
74Float = 3 # e.g. for read -t timeout value
75Bool = 4
76
77
78class _Attributes(object):
79 """Object to hold flags.
80
81 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
82 """
83
84 def __init__(self, defaults):
85 # type: (Dict[str, value_t]) -> None
86
87 # New style
88 self.attrs = {} # type: Dict[str, value_t]
89
90 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
91 self.shopt_changes = [
92 ] # type: List[OptChange] # -O nullglob +O nullglob
93 self.show_options = False # 'set -o' without an argument
94 self.actions = [] # type: List[str] # for compgen -A
95 self.saw_double_dash = False # for set --
96 for name, v in iteritems(defaults):
97 self.Set(name, v)
98
99 def SetTrue(self, name):
100 # type: (str) -> None
101 self.Set(name, value.Bool(True))
102
103 def Set(self, name, val):
104 # type: (str, value_t) -> None
105
106 # debug-completion -> debug_completion
107 name = name.replace('-', '_')
108
109 # similar hack to avoid C++ keyword in frontend/flag_gen.py
110 if name == 'extern':
111 name = 'extern_'
112
113 self.attrs[name] = val
114
115 if 0:
116 # Backward compatibility!
117 with tagswitch(val) as case:
118 if case(value_e.Undef):
119 py_val = None # type: Any
120 elif case(value_e.Bool):
121 py_val = cast(value.Bool, val).b
122 elif case(value_e.Int):
123 py_val = cast(value.Int, val).i
124 elif case(value_e.Float):
125 py_val = cast(value.Float, val).f
126 elif case(value_e.Str):
127 py_val = cast(value.Str, val).s
128 else:
129 raise AssertionError(val)
130
131 setattr(self, name, py_val)
132
133 def __repr__(self):
134 # type: () -> str
135 return '<_Attributes %s>' % self.__dict__
136
137
138class Reader(object):
139 """Wrapper for argv.
140
141 Modified by both the parsing loop and various actions.
142
143 The caller of the flags parser can continue to use it after flag parsing is
144 done to get args.
145 """
146
147 def __init__(self, argv, locs=None):
148 # type: (List[str], Optional[List[CompoundWord]]) -> None
149 self.argv = argv
150 self.locs = locs
151 self.n = len(argv)
152 self.i = 0
153
154 def __repr__(self):
155 # type: () -> str
156 return '<args.Reader %r %d>' % (self.argv, self.i)
157
158 def Next(self):
159 # type: () -> None
160 """Advance."""
161 self.i += 1
162
163 def Peek(self):
164 # type: () -> Optional[str]
165 """Return the next token, or None if there are no more.
166
167 None is your SENTINEL for parsing.
168 """
169 if self.i >= self.n:
170 return None
171 else:
172 return self.argv[self.i]
173
174 def Peek2(self):
175 # type: () -> Tuple[Optional[str], loc_t]
176 """Return the next token, or None if there are no more.
177
178 None is your SENTINEL for parsing.
179 """
180 if self.i >= self.n:
181 return None, loc.Missing
182 else:
183 return self.argv[self.i], self.locs[self.i]
184
185 def ReadRequired(self, error_msg):
186 # type: (str) -> str
187 arg = self.Peek()
188 if arg is None:
189 # point at argv[0]
190 e_usage(error_msg, self._FirstLocation())
191 self.Next()
192 return arg
193
194 def ReadRequired2(self, error_msg):
195 # type: (str) -> Tuple[str, loc_t]
196 arg = self.Peek()
197 if arg is None:
198 # point at argv[0]
199 e_usage(error_msg, self._FirstLocation())
200 location = self.locs[self.i]
201 self.Next()
202 return arg, location
203
204 def Rest(self):
205 # type: () -> List[str]
206 """Return the rest of the arguments."""
207 return self.argv[self.i:]
208
209 def Rest2(self):
210 # type: () -> Tuple[List[str], List[CompoundWord]]
211 """Return the rest of the arguments."""
212 return self.argv[self.i:], self.locs[self.i:]
213
214 def AtEnd(self):
215 # type: () -> bool
216 return self.i >= self.n # must be >= and not ==
217
218 def Done(self):
219 # type: () -> None
220 if not self.AtEnd():
221 e_usage('got too many arguments', self.Location())
222
223 def _FirstLocation(self):
224 # type: () -> loc_t
225 if self.locs is not None and self.locs[0] is not None:
226 return self.locs[0]
227 else:
228 return loc.Missing
229
230 def Location(self):
231 # type: () -> loc_t
232 if self.locs is not None:
233 if self.i == self.n:
234 i = self.n - 1 # if the last arg is missing, point at the one before
235 else:
236 i = self.i
237 if self.locs[i] is not None:
238 return self.locs[i]
239 else:
240 return loc.Missing
241 else:
242 return loc.Missing
243
244
245class _Action(object):
246 """What is done when a flag or option is detected."""
247
248 def __init__(self):
249 # type: () -> None
250 """Empty constructor for mycpp."""
251 pass
252
253 def OnMatch(self, attached_arg, arg_r, out):
254 # type: (Optional[str], Reader, _Attributes) -> bool
255 """Called when the flag matches.
256
257 Args:
258 prefix: '-' or '+'
259 suffix: ',' for -d,
260 arg_r: Reader() (rename to Input or InputReader?)
261 out: _Attributes() -- the thing we want to set
262
263 Returns:
264 True if flag parsing should be aborted.
265 """
266 raise NotImplementedError()
267
268
269class _ArgAction(_Action):
270
271 def __init__(self, name, quit_parsing_flags, valid=None):
272 # type: (str, bool, Optional[List[str]]) -> None
273 """
274 Args:
275 quit_parsing_flags: Stop parsing args after this one. for sh -c.
276 python -c behaves the same way.
277 """
278 self.name = name
279 self.quit_parsing_flags = quit_parsing_flags
280 self.valid = valid
281
282 def _Value(self, arg, location):
283 # type: (str, loc_t) -> value_t
284 raise NotImplementedError()
285
286 def OnMatch(self, attached_arg, arg_r, out):
287 # type: (Optional[str], Reader, _Attributes) -> bool
288 """Called when the flag matches."""
289 if attached_arg is not None: # for the ',' in -d,
290 arg = attached_arg
291 else:
292 arg_r.Next()
293 arg = arg_r.Peek()
294 if arg is None:
295 e_usage('expected argument to %r' % ('-' + self.name),
296 arg_r.Location())
297
298 val = self._Value(arg, arg_r.Location())
299 out.Set(self.name, val)
300 return self.quit_parsing_flags
301
302
303class SetToInt(_ArgAction):
304
305 def __init__(self, name):
306 # type: (str) -> None
307 # repeat defaults for C++ translation
308 _ArgAction.__init__(self, name, False, valid=None)
309
310 def _Value(self, arg, location):
311 # type: (str, loc_t) -> value_t
312 #if match.LooksLikeInteger(arg):
313 if True: # break dependency for prebuilt/
314 ok, i = mops.FromStr2(arg)
315 if not ok:
316 #e_usage('Integer too big: %s' % arg, location)
317 e_usage(
318 'expected integer after %s, got %r' %
319 ('-' + self.name, arg), location)
320 else:
321 pass
322 #e_usage(
323 #'expected integer after %s, got %r' % ('-' + self.name, arg),
324 #location)
325
326 # So far all our int values are > 0, so use -1 as the 'unset' value
327 # corner case: this treats -0 as 0!
328 if mops.Greater(mops.BigInt(0), i):
329 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
330 location)
331 return value.Int(i)
332
333
334class SetToFloat(_ArgAction):
335
336 def __init__(self, name):
337 # type: (str) -> None
338 # repeat defaults for C++ translation
339 _ArgAction.__init__(self, name, False, valid=None)
340
341 def _Value(self, arg, location):
342 # type: (str, loc_t) -> value_t
343 try:
344 f = float(arg)
345 except ValueError:
346 e_usage(
347 'expected number after %r, got %r' % ('-' + self.name, arg),
348 location)
349 # So far all our float values are > 0, so use -1.0 as the 'unset' value
350 # corner case: this treats -0.0 as 0.0!
351 if f < 0:
352 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
353 location)
354 return value.Float(f)
355
356
357class SetToString(_ArgAction):
358
359 def __init__(self, name, quit_parsing_flags, valid=None):
360 # type: (str, bool, Optional[List[str]]) -> None
361 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
362
363 def _Value(self, arg, location):
364 # type: (str, loc_t) -> value_t
365 if self.valid is not None and arg not in self.valid:
366 e_usage(
367 'got invalid argument %r to %r, expected one of: %s' %
368 (arg, ('-' + self.name), '|'.join(self.valid)), location)
369 return value.Str(arg)
370
371
372class SetAttachedBool(_Action):
373
374 def __init__(self, name):
375 # type: (str) -> None
376 self.name = name
377
378 def OnMatch(self, attached_arg, arg_r, out):
379 # type: (Optional[str], Reader, _Attributes) -> bool
380 """Called when the flag matches."""
381
382 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
383 #
384 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
385 #
386 # 'attached_arg' is also used for -t0 though, which is weird
387
388 if attached_arg is not None: # '0' in --verbose=0
389 if attached_arg in ('0', 'F', 'false',
390 'False'): # TODO: incorrect translation
391 b = False
392 elif attached_arg in ('1', 'T', 'true', 'Talse'):
393 b = True
394 else:
395 e_usage(
396 'got invalid argument to boolean flag: %r' % attached_arg,
397 loc.Missing)
398 else:
399 b = True
400
401 out.Set(self.name, value.Bool(b))
402 return False
403
404
405class SetToTrue(_Action):
406
407 def __init__(self, name):
408 # type: (str) -> None
409 self.name = name
410
411 def OnMatch(self, attached_arg, arg_r, out):
412 # type: (Optional[str], Reader, _Attributes) -> bool
413 """Called when the flag matches."""
414 out.SetTrue(self.name)
415 return False
416
417
418class SetOption(_Action):
419 """Set an option to a boolean, for 'set +e'."""
420
421 def __init__(self, name):
422 # type: (str) -> None
423 self.name = name
424
425 def OnMatch(self, attached_arg, arg_r, out):
426 # type: (Optional[str], Reader, _Attributes) -> bool
427 """Called when the flag matches."""
428 b = (attached_arg == '-')
429 out.opt_changes.append((self.name, b))
430 return False
431
432
433class SetNamedOption(_Action):
434 """Set a named option to a boolean, for 'set +o errexit'."""
435
436 def __init__(self, shopt=False):
437 # type: (bool) -> None
438 self.names = [] # type: List[str]
439 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
440
441 def ArgName(self, name):
442 # type: (str) -> None
443 self.names.append(name)
444
445 def OnMatch(self, attached_arg, arg_r, out):
446 # type: (Optional[str], Reader, _Attributes) -> bool
447 """Called when the flag matches."""
448 b = (attached_arg == '-')
449 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
450 arg_r.Next() # always advance
451 arg = arg_r.Peek()
452 if arg is None:
453 # triggers on 'set -O' in addition to 'set -o' (meh OK)
454 out.show_options = True
455 return True # quit parsing
456
457 attr_name = arg # Note: validation is done elsewhere
458 if len(self.names) and attr_name not in self.names:
459 e_usage('Invalid option %r' % arg, loc.Missing)
460 changes = out.shopt_changes if self.shopt else out.opt_changes
461 changes.append((attr_name, b))
462 return False
463
464
465class SetAction(_Action):
466 """For compgen -f."""
467
468 def __init__(self, name):
469 # type: (str) -> None
470 self.name = name
471
472 def OnMatch(self, attached_arg, arg_r, out):
473 # type: (Optional[str], Reader, _Attributes) -> bool
474 out.actions.append(self.name)
475 return False
476
477
478class SetNamedAction(_Action):
479 """For compgen -A file."""
480
481 def __init__(self):
482 # type: () -> None
483 self.names = [] # type: List[str]
484
485 def ArgName(self, name):
486 # type: (str) -> None
487 self.names.append(name)
488
489 def OnMatch(self, attached_arg, arg_r, out):
490 # type: (Optional[str], Reader, _Attributes) -> bool
491 """Called when the flag matches."""
492 arg_r.Next() # always advance
493 arg = arg_r.Peek()
494 if arg is None:
495 e_usage('Expected argument for action', loc.Missing)
496
497 attr_name = arg
498 # Validate the option name against a list of valid names.
499 if len(self.names) and attr_name not in self.names:
500 e_usage('Invalid action name %r' % arg, loc.Missing)
501 out.actions.append(attr_name)
502 return False
503
504
505def Parse(spec, arg_r):
506 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
507
508 # NOTE about -:
509 # 'set -' ignores it, vs set
510 # 'unset -' or 'export -' seems to treat it as a variable name
511 out = _Attributes(spec.defaults)
512
513 while not arg_r.AtEnd():
514 arg = arg_r.Peek()
515 if arg == '--':
516 out.saw_double_dash = True
517 arg_r.Next()
518 break
519
520 # Only accept -- if there are any long flags defined
521 if len(spec.actions_long) and arg.startswith('--'):
522 pos = arg.find('=', 2)
523 if pos == -1:
524 suffix = None # type: Optional[str]
525 flag_name = arg[2:] # strip off --
526 else:
527 suffix = arg[pos + 1:]
528 flag_name = arg[2:pos]
529
530 action = spec.actions_long.get(flag_name)
531 if action is None:
532 e_usage('got invalid flag %r' % arg, arg_r.Location())
533
534 action.OnMatch(suffix, arg_r, out)
535 arg_r.Next()
536 continue
537
538 elif arg.startswith('-') and len(arg) > 1:
539 n = len(arg)
540 for i in xrange(1, n): # parse flag combos like -rx
541 ch = arg[i]
542
543 if ch == '0':
544 ch = 'Z' # hack for read -0
545
546 if ch in spec.plus_flags:
547 out.Set(ch, value.Str('-'))
548 continue
549
550 if ch in spec.arity0: # e.g. read -r
551 out.SetTrue(ch)
552 continue
553
554 if ch in spec.arity1: # e.g. read -t1.0
555 action = spec.arity1[ch]
556 # make sure we don't pass empty string for read -t
557 attached_arg = arg[i + 1:] if i < n - 1 else None
558 action.OnMatch(attached_arg, arg_r, out)
559 break
560
561 e_usage("doesn't accept flag %s" % ('-' + ch),
562 arg_r.Location())
563
564 arg_r.Next() # next arg
565
566 # Only accept + if there are ANY options defined, e.g. for declare +rx.
567 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
568 n = len(arg)
569 for i in xrange(1, n): # parse flag combos like -rx
570 ch = arg[i]
571 if ch in spec.plus_flags:
572 out.Set(ch, value.Str('+'))
573 continue
574
575 e_usage("doesn't accept option %s" % ('+' + ch),
576 arg_r.Location())
577
578 arg_r.Next() # next arg
579
580 else: # a regular arg
581 break
582
583 return out
584
585
586def ParseLikeEcho(spec, arg_r):
587 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
588 """Echo is a special case. These work: echo -n echo -en.
589
590 - But don't respect --
591 - doesn't fail when an invalid flag is passed
592 """
593 out = _Attributes(spec.defaults)
594
595 while not arg_r.AtEnd():
596 arg = arg_r.Peek()
597 chars = arg[1:]
598 if arg.startswith('-') and len(chars):
599 # Check if it looks like -en or not. TODO: could optimize this.
600 done = False
601 for c in chars:
602 if c not in spec.arity0:
603 done = True
604 break
605 if done:
606 break
607
608 for ch in chars:
609 out.SetTrue(ch)
610
611 else:
612 break # Looks like an arg
613
614 arg_r.Next() # next arg
615
616 return out
617
618
619def ParseMore(spec, arg_r):
620 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
621 """Return attributes and an index.
622
623 Respects +, like set +eu
624
625 We do NOT respect:
626
627 WRONG: sh -cecho OK: sh -c echo
628 WRONG: set -opipefail OK: set -o pipefail
629
630 But we do accept these
631
632 set -euo pipefail
633 set -oeu pipefail
634 set -oo pipefail errexit
635 """
636 out = _Attributes(spec.defaults)
637
638 quit = False
639 while not arg_r.AtEnd():
640 arg = arg_r.Peek()
641 if arg == '--':
642 out.saw_double_dash = True
643 arg_r.Next()
644 break
645
646 if arg.startswith('--'):
647 action = spec.actions_long.get(arg[2:])
648 if action is None:
649 e_usage('got invalid flag %r' % arg, arg_r.Location())
650
651 # Note: not parsing --foo=bar as attached_arg, as above
652 action.OnMatch(None, arg_r, out)
653 arg_r.Next()
654 continue
655
656 # corner case: sh +c is also accepted!
657 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
658 # note: we're not handling sh -cecho (no space) as an argument
659 # It complains about a missing argument
660
661 char0 = arg[0]
662
663 # TODO: set - - empty
664 for ch in arg[1:]:
665 #log('ch %r arg_r %s', ch, arg_r)
666 action = spec.actions_short.get(ch)
667 if action is None:
668 e_usage('got invalid flag %r' % ('-' + ch),
669 arg_r.Location())
670
671 attached_arg = char0 if ch in spec.plus_flags else None
672 quit = action.OnMatch(attached_arg, arg_r, out)
673 arg_r.Next() # process the next flag
674
675 if quit:
676 break
677 else:
678 continue
679
680 break # it's a regular arg
681
682 return out