OILS / frontend / args.py View on Github | oilshell.org

675 lines, 348 significant
1"""
2args.py - Flag, option, and arg parsing for the shell.
3
4All existing shells have their own flag parsing, rather than using libc.
5
6We have 3 types of flag parsing here:
7
8 FlagSpecAndMore() -- e.g. for 'sh +u -o errexit' and 'set +u -o errexit'
9 FlagSpec() -- for echo -en, read -t1.0, etc.
10
11Examples:
12 set -opipefail # not allowed, space required
13 read -t1.0 # allowed
14
15Things that getopt/optparse don't support:
16
17- accepts +o +n for 'set' and bin/osh
18 - pushd and popd also uses +, although it's not an arg.
19- parses args -- well argparse is supposed to do this
20- maybe: integrate with usage
21- maybe: integrate with flags
22
23optparse:
24 - has option groups (Go flag package has flagset)
25
26NOTES about builtins:
27- eval and echo implicitly join their args. We don't want that.
28 - option strict-eval and strict-echo
29- bash is inconsistent about checking for extra args
30 - exit 1 2 complains, but pushd /lib /bin just ignores second argument
31 - it has a no_args() function that isn't called everywhere. It's not
32 declarative.
33
34TODO:
35 - Autogenerate help from help='' fields. Usage line like FlagSpec('echo [-en]')
36
37GNU notes:
38
39- Consider adding GNU-style option to interleave flags and args?
40 - Not sure I like this.
41- GNU getopt has fuzzy matching for long flags. I think we should rely
42 on good completion instead.
43
44Bash notes:
45
46bashgetopt.c codes:
47 leading +: allow options
48 : requires argument
49 ; argument may be missing
50 # numeric argument
51
52However I don't see these used anywhere! I only see ':' used.
53"""
54from __future__ import print_function
55
56from _devbuild.gen.syntax_asdl import loc, loc_t, CompoundWord
57from _devbuild.gen.value_asdl import (value, value_e, value_t)
58
59from core.error import e_usage
60from frontend import match
61from mycpp import mops
62from mycpp.mylib import log, tagswitch, iteritems
63
64_ = log
65
66from typing import (cast, Tuple, Optional, Dict, List, Any, TYPE_CHECKING)
67if TYPE_CHECKING:
68 from frontend import flag_spec
69 OptChange = Tuple[str, bool]
70
71# TODO: Move to flag_spec? We use flag_type_t
72String = 1
73Int = 2
74Float = 3 # e.g. for read -t timeout value
75Bool = 4
76
77
78class _Attributes(object):
79 """Object to hold flags.
80
81 TODO: FlagSpec doesn't need this; only FlagSpecAndMore.
82 """
83
84 def __init__(self, defaults):
85 # type: (Dict[str, value_t]) -> None
86
87 # New style
88 self.attrs = {} # type: Dict[str, value_t]
89
90 self.opt_changes = [] # type: List[OptChange] # -o errexit +o nounset
91 self.shopt_changes = [
92 ] # type: List[OptChange] # -O nullglob +O nullglob
93 self.show_options = False # 'set -o' without an argument
94 self.actions = [] # type: List[str] # for compgen -A
95 self.saw_double_dash = False # for set --
96 for name, v in iteritems(defaults):
97 self.Set(name, v)
98
99 def SetTrue(self, name):
100 # type: (str) -> None
101 self.Set(name, value.Bool(True))
102
103 def Set(self, name, val):
104 # type: (str, value_t) -> None
105
106 # debug-completion -> debug_completion
107 name = name.replace('-', '_')
108
109 # similar hack to avoid C++ keyword in frontend/flag_gen.py
110 if name == 'extern':
111 name = 'extern_'
112
113 self.attrs[name] = val
114
115 if 0:
116 # Backward compatibility!
117 with tagswitch(val) as case:
118 if case(value_e.Undef):
119 py_val = None # type: Any
120 elif case(value_e.Bool):
121 py_val = cast(value.Bool, val).b
122 elif case(value_e.Int):
123 py_val = cast(value.Int, val).i
124 elif case(value_e.Float):
125 py_val = cast(value.Float, val).f
126 elif case(value_e.Str):
127 py_val = cast(value.Str, val).s
128 else:
129 raise AssertionError(val)
130
131 setattr(self, name, py_val)
132
133 def __repr__(self):
134 # type: () -> str
135 return '<_Attributes %s>' % self.__dict__
136
137
138class Reader(object):
139 """Wrapper for argv.
140
141 Modified by both the parsing loop and various actions.
142
143 The caller of the flags parser can continue to use it after flag parsing is
144 done to get args.
145 """
146
147 def __init__(self, argv, locs=None):
148 # type: (List[str], Optional[List[CompoundWord]]) -> None
149 self.argv = argv
150 self.locs = locs
151 self.n = len(argv)
152 self.i = 0
153
154 def __repr__(self):
155 # type: () -> str
156 return '<args.Reader %r %d>' % (self.argv, self.i)
157
158 def Next(self):
159 # type: () -> None
160 """Advance."""
161 self.i += 1
162
163 def Peek(self):
164 # type: () -> Optional[str]
165 """Return the next token, or None if there are no more.
166
167 None is your SENTINEL for parsing.
168 """
169 if self.i >= self.n:
170 return None
171 else:
172 return self.argv[self.i]
173
174 def Peek2(self):
175 # type: () -> Tuple[Optional[str], loc_t]
176 """Return the next token, or None if there are no more.
177
178 None is your SENTINEL for parsing.
179 """
180 if self.i >= self.n:
181 return None, loc.Missing
182 else:
183 return self.argv[self.i], self.locs[self.i]
184
185 def ReadRequired(self, error_msg):
186 # type: (str) -> str
187 arg = self.Peek()
188 if arg is None:
189 # point at argv[0]
190 e_usage(error_msg, self._FirstLocation())
191 self.Next()
192 return arg
193
194 def ReadRequired2(self, error_msg):
195 # type: (str) -> Tuple[str, loc_t]
196 arg = self.Peek()
197 if arg is None:
198 # point at argv[0]
199 e_usage(error_msg, self._FirstLocation())
200 location = self.locs[self.i]
201 self.Next()
202 return arg, location
203
204 def Rest(self):
205 # type: () -> List[str]
206 """Return the rest of the arguments."""
207 return self.argv[self.i:]
208
209 def Rest2(self):
210 # type: () -> Tuple[List[str], List[CompoundWord]]
211 """Return the rest of the arguments."""
212 return self.argv[self.i:], self.locs[self.i:]
213
214 def AtEnd(self):
215 # type: () -> bool
216 return self.i >= self.n # must be >= and not ==
217
218 def Done(self):
219 # type: () -> None
220 if not self.AtEnd():
221 e_usage('got too many arguments', self.Location())
222
223 def _FirstLocation(self):
224 # type: () -> loc_t
225 if self.locs is not None and self.locs[0] is not None:
226 return self.locs[0]
227 else:
228 return loc.Missing
229
230 def Location(self):
231 # type: () -> loc_t
232 if self.locs is not None:
233 if self.i == self.n:
234 i = self.n - 1 # if the last arg is missing, point at the one before
235 else:
236 i = self.i
237 if self.locs[i] is not None:
238 return self.locs[i]
239 else:
240 return loc.Missing
241 else:
242 return loc.Missing
243
244
245class _Action(object):
246 """What is done when a flag or option is detected."""
247
248 def __init__(self):
249 # type: () -> None
250 """Empty constructor for mycpp."""
251 pass
252
253 def OnMatch(self, attached_arg, arg_r, out):
254 # type: (Optional[str], Reader, _Attributes) -> bool
255 """Called when the flag matches.
256
257 Args:
258 prefix: '-' or '+'
259 suffix: ',' for -d,
260 arg_r: Reader() (rename to Input or InputReader?)
261 out: _Attributes() -- the thing we want to set
262
263 Returns:
264 True if flag parsing should be aborted.
265 """
266 raise NotImplementedError()
267
268
269class _ArgAction(_Action):
270
271 def __init__(self, name, quit_parsing_flags, valid=None):
272 # type: (str, bool, Optional[List[str]]) -> None
273 """
274 Args:
275 quit_parsing_flags: Stop parsing args after this one. for sh -c.
276 python -c behaves the same way.
277 """
278 self.name = name
279 self.quit_parsing_flags = quit_parsing_flags
280 self.valid = valid
281
282 def _Value(self, arg, location):
283 # type: (str, loc_t) -> value_t
284 raise NotImplementedError()
285
286 def OnMatch(self, attached_arg, arg_r, out):
287 # type: (Optional[str], Reader, _Attributes) -> bool
288 """Called when the flag matches."""
289 if attached_arg is not None: # for the ',' in -d,
290 arg = attached_arg
291 else:
292 arg_r.Next()
293 arg = arg_r.Peek()
294 if arg is None:
295 e_usage('expected argument to %r' % ('-' + self.name),
296 arg_r.Location())
297
298 val = self._Value(arg, arg_r.Location())
299 out.Set(self.name, val)
300 return self.quit_parsing_flags
301
302
303class SetToInt(_ArgAction):
304
305 def __init__(self, name):
306 # type: (str) -> None
307 # repeat defaults for C++ translation
308 _ArgAction.__init__(self, name, False, valid=None)
309
310 def _Value(self, arg, location):
311 # type: (str, loc_t) -> value_t
312 if match.LooksLikeInteger(arg):
313 i = mops.FromStr(arg)
314 else:
315 e_usage(
316 'expected integer after %s, got %r' % ('-' + self.name, arg),
317 location)
318
319 # So far all our int values are > 0, so use -1 as the 'unset' value
320 # corner case: this treats -0 as 0!
321 if mops.Greater(mops.BigInt(0), i):
322 e_usage('got invalid integer for %s: %s' % ('-' + self.name, arg),
323 location)
324 return value.Int(i)
325
326
327class SetToFloat(_ArgAction):
328
329 def __init__(self, name):
330 # type: (str) -> None
331 # repeat defaults for C++ translation
332 _ArgAction.__init__(self, name, False, valid=None)
333
334 def _Value(self, arg, location):
335 # type: (str, loc_t) -> value_t
336 try:
337 f = float(arg)
338 except ValueError:
339 e_usage(
340 'expected number after %r, got %r' % ('-' + self.name, arg),
341 location)
342 # So far all our float values are > 0, so use -1.0 as the 'unset' value
343 # corner case: this treats -0.0 as 0.0!
344 if f < 0:
345 e_usage('got invalid float for %s: %s' % ('-' + self.name, arg),
346 location)
347 return value.Float(f)
348
349
350class SetToString(_ArgAction):
351
352 def __init__(self, name, quit_parsing_flags, valid=None):
353 # type: (str, bool, Optional[List[str]]) -> None
354 _ArgAction.__init__(self, name, quit_parsing_flags, valid=valid)
355
356 def _Value(self, arg, location):
357 # type: (str, loc_t) -> value_t
358 if self.valid is not None and arg not in self.valid:
359 e_usage(
360 'got invalid argument %r to %r, expected one of: %s' %
361 (arg, ('-' + self.name), '|'.join(self.valid)), location)
362 return value.Str(arg)
363
364
365class SetAttachedBool(_Action):
366
367 def __init__(self, name):
368 # type: (str) -> None
369 self.name = name
370
371 def OnMatch(self, attached_arg, arg_r, out):
372 # type: (Optional[str], Reader, _Attributes) -> bool
373 """Called when the flag matches."""
374
375 # TODO: Delete this part? Is this eqvuivalent to SetToTrue?
376 #
377 # We're not using Go-like --verbose=1, --verbose, or --verbose=0
378 #
379 # 'attached_arg' is also used for -t0 though, which is weird
380
381 if attached_arg is not None: # '0' in --verbose=0
382 if attached_arg in ('0', 'F', 'false',
383 'False'): # TODO: incorrect translation
384 b = False
385 elif attached_arg in ('1', 'T', 'true', 'Talse'):
386 b = True
387 else:
388 e_usage(
389 'got invalid argument to boolean flag: %r' % attached_arg,
390 loc.Missing)
391 else:
392 b = True
393
394 out.Set(self.name, value.Bool(b))
395 return False
396
397
398class SetToTrue(_Action):
399
400 def __init__(self, name):
401 # type: (str) -> None
402 self.name = name
403
404 def OnMatch(self, attached_arg, arg_r, out):
405 # type: (Optional[str], Reader, _Attributes) -> bool
406 """Called when the flag matches."""
407 out.SetTrue(self.name)
408 return False
409
410
411class SetOption(_Action):
412 """Set an option to a boolean, for 'set +e'."""
413
414 def __init__(self, name):
415 # type: (str) -> None
416 self.name = name
417
418 def OnMatch(self, attached_arg, arg_r, out):
419 # type: (Optional[str], Reader, _Attributes) -> bool
420 """Called when the flag matches."""
421 b = (attached_arg == '-')
422 out.opt_changes.append((self.name, b))
423 return False
424
425
426class SetNamedOption(_Action):
427 """Set a named option to a boolean, for 'set +o errexit'."""
428
429 def __init__(self, shopt=False):
430 # type: (bool) -> None
431 self.names = [] # type: List[str]
432 self.shopt = shopt # is it sh -o (set) or sh -O (shopt)?
433
434 def ArgName(self, name):
435 # type: (str) -> None
436 self.names.append(name)
437
438 def OnMatch(self, attached_arg, arg_r, out):
439 # type: (Optional[str], Reader, _Attributes) -> bool
440 """Called when the flag matches."""
441 b = (attached_arg == '-')
442 #log('SetNamedOption %r %r %r', prefix, suffix, arg_r)
443 arg_r.Next() # always advance
444 arg = arg_r.Peek()
445 if arg is None:
446 # triggers on 'set -O' in addition to 'set -o' (meh OK)
447 out.show_options = True
448 return True # quit parsing
449
450 attr_name = arg # Note: validation is done elsewhere
451 if len(self.names) and attr_name not in self.names:
452 e_usage('Invalid option %r' % arg, loc.Missing)
453 changes = out.shopt_changes if self.shopt else out.opt_changes
454 changes.append((attr_name, b))
455 return False
456
457
458class SetAction(_Action):
459 """For compgen -f."""
460
461 def __init__(self, name):
462 # type: (str) -> None
463 self.name = name
464
465 def OnMatch(self, attached_arg, arg_r, out):
466 # type: (Optional[str], Reader, _Attributes) -> bool
467 out.actions.append(self.name)
468 return False
469
470
471class SetNamedAction(_Action):
472 """For compgen -A file."""
473
474 def __init__(self):
475 # type: () -> None
476 self.names = [] # type: List[str]
477
478 def ArgName(self, name):
479 # type: (str) -> None
480 self.names.append(name)
481
482 def OnMatch(self, attached_arg, arg_r, out):
483 # type: (Optional[str], Reader, _Attributes) -> bool
484 """Called when the flag matches."""
485 arg_r.Next() # always advance
486 arg = arg_r.Peek()
487 if arg is None:
488 e_usage('Expected argument for action', loc.Missing)
489
490 attr_name = arg
491 # Validate the option name against a list of valid names.
492 if len(self.names) and attr_name not in self.names:
493 e_usage('Invalid action name %r' % arg, loc.Missing)
494 out.actions.append(attr_name)
495 return False
496
497
498def Parse(spec, arg_r):
499 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
500
501 # NOTE about -:
502 # 'set -' ignores it, vs set
503 # 'unset -' or 'export -' seems to treat it as a variable name
504 out = _Attributes(spec.defaults)
505
506 while not arg_r.AtEnd():
507 arg = arg_r.Peek()
508 if arg == '--':
509 out.saw_double_dash = True
510 arg_r.Next()
511 break
512
513 # Only accept -- if there are any long flags defined
514 if len(spec.actions_long) and arg.startswith('--'):
515 pos = arg.find('=', 2)
516 if pos == -1:
517 suffix = None # type: Optional[str]
518 flag_name = arg[2:] # strip off --
519 else:
520 suffix = arg[pos + 1:]
521 flag_name = arg[2:pos]
522
523 action = spec.actions_long.get(flag_name)
524 if action is None:
525 e_usage('got invalid flag %r' % arg, arg_r.Location())
526
527 action.OnMatch(suffix, arg_r, out)
528 arg_r.Next()
529 continue
530
531 elif arg.startswith('-') and len(arg) > 1:
532 n = len(arg)
533 for i in xrange(1, n): # parse flag combos like -rx
534 ch = arg[i]
535
536 if ch == '0':
537 ch = 'Z' # hack for read -0
538
539 if ch in spec.plus_flags:
540 out.Set(ch, value.Str('-'))
541 continue
542
543 if ch in spec.arity0: # e.g. read -r
544 out.SetTrue(ch)
545 continue
546
547 if ch in spec.arity1: # e.g. read -t1.0
548 action = spec.arity1[ch]
549 # make sure we don't pass empty string for read -t
550 attached_arg = arg[i + 1:] if i < n - 1 else None
551 action.OnMatch(attached_arg, arg_r, out)
552 break
553
554 e_usage("doesn't accept flag %s" % ('-' + ch),
555 arg_r.Location())
556
557 arg_r.Next() # next arg
558
559 # Only accept + if there are ANY options defined, e.g. for declare +rx.
560 elif len(spec.plus_flags) and arg.startswith('+') and len(arg) > 1:
561 n = len(arg)
562 for i in xrange(1, n): # parse flag combos like -rx
563 ch = arg[i]
564 if ch in spec.plus_flags:
565 out.Set(ch, value.Str('+'))
566 continue
567
568 e_usage("doesn't accept option %s" % ('+' + ch),
569 arg_r.Location())
570
571 arg_r.Next() # next arg
572
573 else: # a regular arg
574 break
575
576 return out
577
578
579def ParseLikeEcho(spec, arg_r):
580 # type: (flag_spec._FlagSpec, Reader) -> _Attributes
581 """Echo is a special case. These work: echo -n echo -en.
582
583 - But don't respect --
584 - doesn't fail when an invalid flag is passed
585 """
586 out = _Attributes(spec.defaults)
587
588 while not arg_r.AtEnd():
589 arg = arg_r.Peek()
590 chars = arg[1:]
591 if arg.startswith('-') and len(chars):
592 # Check if it looks like -en or not. TODO: could optimize this.
593 done = False
594 for c in chars:
595 if c not in spec.arity0:
596 done = True
597 break
598 if done:
599 break
600
601 for ch in chars:
602 out.SetTrue(ch)
603
604 else:
605 break # Looks like an arg
606
607 arg_r.Next() # next arg
608
609 return out
610
611
612def ParseMore(spec, arg_r):
613 # type: (flag_spec._FlagSpecAndMore, Reader) -> _Attributes
614 """Return attributes and an index.
615
616 Respects +, like set +eu
617
618 We do NOT respect:
619
620 WRONG: sh -cecho OK: sh -c echo
621 WRONG: set -opipefail OK: set -o pipefail
622
623 But we do accept these
624
625 set -euo pipefail
626 set -oeu pipefail
627 set -oo pipefail errexit
628 """
629 out = _Attributes(spec.defaults)
630
631 quit = False
632 while not arg_r.AtEnd():
633 arg = arg_r.Peek()
634 if arg == '--':
635 out.saw_double_dash = True
636 arg_r.Next()
637 break
638
639 if arg.startswith('--'):
640 action = spec.actions_long.get(arg[2:])
641 if action is None:
642 e_usage('got invalid flag %r' % arg, arg_r.Location())
643
644 # Note: not parsing --foo=bar as attached_arg, as above
645 action.OnMatch(None, arg_r, out)
646 arg_r.Next()
647 continue
648
649 # corner case: sh +c is also accepted!
650 if (arg.startswith('-') or arg.startswith('+')) and len(arg) > 1:
651 # note: we're not handling sh -cecho (no space) as an argument
652 # It complains about a missing argument
653
654 char0 = arg[0]
655
656 # TODO: set - - empty
657 for ch in arg[1:]:
658 #log('ch %r arg_r %s', ch, arg_r)
659 action = spec.actions_short.get(ch)
660 if action is None:
661 e_usage('got invalid flag %r' % ('-' + ch),
662 arg_r.Location())
663
664 attached_arg = char0 if ch in spec.plus_flags else None
665 quit = action.OnMatch(attached_arg, arg_r, out)
666 arg_r.Next() # process the next flag
667
668 if quit:
669 break
670 else:
671 continue
672
673 break # it's a regular arg
674
675 return out