OILS / core / vm.py View on Github | oils.pub

542 lines, 306 significant
1"""vm.py - interfaces for executing shell."""
2from __future__ import print_function
3
4from _devbuild.gen.id_kind_asdl import Id, Id_t, Id_str
5from _devbuild.gen.option_asdl import builtin_i
6from _devbuild.gen.runtime_asdl import (CommandStatus, StatusArray, flow_e,
7 flow_t)
8from _devbuild.gen.syntax_asdl import Token, loc, loc_t
9from _devbuild.gen.value_asdl import value, value_e, value_t, Obj
10from core import dev
11from core import error
12from core.error import e_die
13from core import pyos
14from core import pyutil
15from display import ui
16from mycpp.mylib import log, tagswitch
17
18from typing import List, Dict, Tuple, Optional, Any, cast, TYPE_CHECKING
19if TYPE_CHECKING:
20 from _devbuild.gen.runtime_asdl import cmd_value, RedirValue
21 from _devbuild.gen.syntax_asdl import (command, command_t, CommandSub)
22 from builtin import hay_ysh
23 from core import optview
24 from core import state
25 from frontend import typed_args
26 from osh import sh_expr_eval
27 from osh.sh_expr_eval import ArithEvaluator
28 from osh.sh_expr_eval import BoolEvaluator
29 from osh import word_eval
30 from osh import cmd_eval
31 from osh import prompt
32 from ysh import expr_eval
33
34_ = log
35
36
37class ControlFlow(Exception):
38 """Internal exception for control flow.
39
40 Used by CommandEvaluator and 'source' builtin
41
42 break and continue are caught by loops, return is caught by functions.
43 """
44 pass
45
46
47class IntControlFlow(Exception):
48
49 def __init__(self, keyword_id, keyword_str, keyword_loc, arg):
50 # type: (Id_t, str, loc_t, int) -> None
51 """
52 Args:
53 token: the keyword token
54 arg: exit code to 'return', or number of levels to break/continue
55 """
56 self.keyword_id = keyword_id
57 self.keyword_str = keyword_str
58 self.keyword_loc = keyword_loc
59 self.arg = arg
60
61 def Keyword(self):
62 # type: () -> str
63 return self.keyword_str
64
65 def Location(self):
66 # type: () -> loc_t
67 return self.keyword_loc
68
69 def IsReturn(self):
70 # type: () -> bool
71 return self.keyword_id == Id.ControlFlow_Return
72
73 def IsBreak(self):
74 # type: () -> bool
75 return self.keyword_id == Id.ControlFlow_Break
76
77 def IsContinue(self):
78 # type: () -> bool
79 return self.keyword_id == Id.ControlFlow_Continue
80
81 def StatusCode(self):
82 # type: () -> int
83 assert self.IsReturn()
84 # All shells except dash do this truncation.
85 # turn 257 into 1, and -1 into 255.
86 return self.arg & 0xff
87
88 def HandleLoop(self):
89 # type: () -> flow_t
90 """Mutates this exception and returns what the caller should do."""
91
92 if self.IsBreak():
93 self.arg -= 1
94 if self.arg == 0:
95 return flow_e.Break # caller should break out of loop
96
97 elif self.IsContinue():
98 self.arg -= 1
99 if self.arg == 0:
100 return flow_e.Nothing # do nothing to continue
101
102 # return / break 2 / continue 2 need to pop up more
103 return flow_e.Raise
104
105 def __repr__(self):
106 # type: () -> str
107 return '<IntControlFlow %s %s>' % (Id_str(self.keyword_id), self.arg)
108
109
110class ValueControlFlow(Exception):
111
112 def __init__(self, token, value):
113 # type: (Token, value_t) -> None
114 """
115 Args:
116 token: the keyword token
117 value: value_t to 'return' from a function
118 """
119 self.token = token
120 self.value = value
121
122 def __repr__(self):
123 # type: () -> str
124 return '<ValueControlFlow %s %s>' % (self.token, self.value)
125
126
127def InitUnsafeArith(mem, word_ev, unsafe_arith):
128 # type: (state.Mem, word_eval.NormalWordEvaluator, sh_expr_eval.UnsafeArith) -> None
129 """Wire up circular dependencies for UnsafeArith."""
130 mem.unsafe_arith = unsafe_arith # for 'declare -n' nameref expansion of a[i]
131 word_ev.unsafe_arith = unsafe_arith # for ${!ref} expansion of a[i]
132
133
134def InitCircularDeps(
135 arith_ev, # type: ArithEvaluator
136 bool_ev, # type: BoolEvaluator
137 expr_ev, # type: expr_eval.ExprEvaluator
138 word_ev, # type: word_eval.NormalWordEvaluator
139 cmd_ev, # type: cmd_eval.CommandEvaluator
140 shell_ex, # type: _Executor
141 pure_ex, # type: _Executor
142 prompt_ev, # type: prompt.Evaluator
143 global_io, # type: Obj
144 tracer, # type: dev.Tracer
145):
146 # type: (...) -> None
147 """Wire up mutually recursive evaluators and runtime objects."""
148 arith_ev.word_ev = word_ev
149 bool_ev.word_ev = word_ev
150
151 if expr_ev: # for pure OSH
152 expr_ev.shell_ex = shell_ex
153 expr_ev.cmd_ev = cmd_ev
154 expr_ev.word_ev = word_ev
155
156 word_ev.arith_ev = arith_ev
157 word_ev.expr_ev = expr_ev
158 word_ev.prompt_ev = prompt_ev
159 word_ev.shell_ex = shell_ex
160
161 cmd_ev.shell_ex = shell_ex
162 cmd_ev.arith_ev = arith_ev
163 cmd_ev.bool_ev = bool_ev
164 cmd_ev.expr_ev = expr_ev
165 cmd_ev.word_ev = word_ev
166 cmd_ev.tracer = tracer
167
168 shell_ex.cmd_ev = cmd_ev
169 pure_ex.cmd_ev = cmd_ev
170
171 prompt_ev.word_ev = word_ev
172 prompt_ev.expr_ev = expr_ev
173 prompt_ev.global_io = global_io
174
175 tracer.word_ev = word_ev
176
177 arith_ev.CheckCircularDeps()
178 bool_ev.CheckCircularDeps()
179 if expr_ev:
180 expr_ev.CheckCircularDeps()
181 word_ev.CheckCircularDeps()
182 cmd_ev.CheckCircularDeps()
183 shell_ex.CheckCircularDeps()
184 pure_ex.CheckCircularDeps()
185 prompt_ev.CheckCircularDeps()
186 tracer.CheckCircularDeps()
187
188
189class _Executor(object):
190
191 def __init__(
192 self,
193 mem, # type: state.Mem
194 exec_opts, # type: optview.Exec
195 mutable_opts, # type: state.MutableOpts
196 procs, # type: state.Procs
197 hay_state, # type: hay_ysh.HayState
198 builtins, # type: Dict[int, _Builtin]
199 tracer, # type: dev.Tracer
200 errfmt # type: ui.ErrorFormatter
201 ):
202 self.mem = mem
203 self.exec_opts = exec_opts
204 self.mutable_opts = mutable_opts # for IsDisabled(), not mutating
205 self.procs = procs
206 self.hay_state = hay_state
207 self.builtins = builtins
208 self.tracer = tracer
209 self.errfmt = errfmt
210
211 # Not a constructor argument
212 self.cmd_ev = None # type: cmd_eval.CommandEvaluator
213
214 def CheckCircularDeps(self):
215 # type: () -> None
216 assert self.cmd_ev is not None
217
218 def RunSimpleCommand(self, cmd_val, cmd_st, run_flags):
219 # type: (cmd_value.Argv, CommandStatus, int) -> int
220 """Shared between ShellExecutor and PureExecutor"""
221 if len(cmd_val.arg_locs):
222 arg0_loc = cmd_val.arg_locs[0] # type: loc_t
223 else:
224 arg0_loc = loc.Missing
225
226 argv = cmd_val.argv
227 # This happens when you write "$@" but have no arguments.
228 if len(argv) == 0:
229 if self.exec_opts.strict_argv():
230 e_die("Command evaluated to an empty argv array", arg0_loc)
231 else:
232 return self.mem.LastStatus() # keep the previous status code
233
234 return self._RunSimpleCommand(argv[0], arg0_loc, cmd_val, cmd_st,
235 run_flags)
236
237 def _RunSimpleCommand(self, arg0, arg0_loc, cmd_val, cmd_st, run_flags):
238 # type: (str, loc_t, cmd_value.Argv, CommandStatus, int) -> int
239 raise NotImplementedError()
240
241 def RunExternal(self, arg0, arg0_loc, cmd_val, cmd_st, run_flags):
242 # type: (str, loc_t, cmd_value.Argv, CommandStatus, int) -> int
243 raise NotImplementedError()
244
245 def RunBuiltin(self, builtin_id, cmd_val):
246 # type: (int, cmd_value.Argv) -> int
247 """Run a builtin.
248
249 Also called by the 'builtin' builtin, in builtin/meta_oils.py
250 """
251 # TODO: We could move this check to LookupSpecialBuiltin, etc.
252 # The error message would be better
253 # This also relates to __builtin__ reflection and so forth
254 if (self.exec_opts.no_osh_builtins() and
255 builtin_id in (builtin_i.alias, builtin_i.unalias,
256 builtin_i.chdir, builtin_i.colon)):
257 if builtin_id == builtin_i.chdir:
258 msg = "Use 'cd' instead of 'chdir' in YSH (no_osh_builtins)"
259 else:
260 msg = ("The %r builtin isn't part of YSH (no_osh_builtins)" %
261 cmd_val.argv[0])
262 e_die(msg, cmd_val.arg_locs[0])
263
264 self.tracer.OnBuiltin(builtin_id, cmd_val.argv)
265 builtin_proc = self.builtins[builtin_id]
266 return self._RunBuiltinProc(builtin_proc, cmd_val)
267
268 def _RunBuiltinProc(self, builtin_proc, cmd_val):
269 # type: (_Builtin, cmd_value.Argv) -> int
270
271 io_errors = [] # type: List[error.IOError_OSError]
272 with ctx_FlushStdout(io_errors):
273 # note: could be second word, like 'builtin read'
274 with ui.ctx_Location(self.errfmt, cmd_val.arg_locs[0]):
275 try:
276 status = builtin_proc.Run(cmd_val)
277 assert isinstance(status, int)
278 except (IOError, OSError) as e:
279 self.errfmt.PrintMessage(
280 '%s builtin I/O error: %s' %
281 (cmd_val.argv[0], pyutil.strerror(e)),
282 cmd_val.arg_locs[0])
283 return 1
284 except error.Usage as e:
285 arg0 = cmd_val.argv[0]
286 # e.g. 'type' doesn't accept flag '-x'
287 self.errfmt.PrefixPrint(e.msg, '%r ' % arg0, e.location)
288 return 2 # consistent error code for usage error
289
290 if len(io_errors): # e.g. disk full, ulimit
291 self.errfmt.PrintMessage(
292 '%s builtin I/O error: %s' %
293 (cmd_val.argv[0], pyutil.strerror(io_errors[0])),
294 cmd_val.arg_locs[0])
295 return 1
296
297 return status
298
299 def _RunInvokable(self, proc_val, self_obj, arg0_loc, cmd_val):
300 # type: (value_t, Optional[Obj], loc_t, cmd_value.Argv) -> int
301
302 cmd_val.self_obj = self_obj # MAYBE bind self
303
304 if self.exec_opts.strict_errexit():
305 disabled_tok = self.mutable_opts.ErrExitDisabledToken()
306 if disabled_tok:
307 self.errfmt.Print_('errexit was disabled for this construct',
308 disabled_tok)
309 self.errfmt.StderrLine('')
310 e_die(
311 "Can't run functions or procs while errexit is disabled (OILS-ERR-301)",
312 arg0_loc)
313
314 with tagswitch(proc_val) as case:
315 if case(value_e.BuiltinProc):
316 # Handle the special case of the BUILTIN proc
317 # module_ysh.ModuleInvoke, which is returned on the Obj
318 # created by 'use util.ysh'
319 builtin_proc = cast(value.BuiltinProc, proc_val)
320 b = cast(_Builtin, builtin_proc.builtin)
321 status = self._RunBuiltinProc(b, cmd_val)
322
323 elif case(value_e.Proc):
324 proc = cast(value.Proc, proc_val)
325 with dev.ctx_Tracer(self.tracer, 'proc', cmd_val.argv):
326 # NOTE: Functions could call 'exit 42' directly, etc.
327 status = self.cmd_ev.RunProc(proc, cmd_val)
328
329 else:
330 # GetInvokable() should only return 1 of 2 things
331 raise AssertionError()
332
333 return status
334
335 def RunBackgroundJob(self, node):
336 # type: (command_t) -> int
337 return 0
338
339 def RunPipeline(self, node, status_out):
340 # type: (command.Pipeline, CommandStatus) -> None
341 pass
342
343 def RunSubshell(self, node):
344 # type: (command_t) -> int
345 return 0
346
347 def CaptureStdout(self, node):
348 # type: (command_t) -> Tuple[int, str]
349 return 0, ''
350
351 def Capture3(self, node):
352 # type: (command_t) -> Tuple[int, str, str]
353 return 0, '', ''
354
355 def RunCommandSub(self, cs_part):
356 # type: (CommandSub) -> str
357 return ''
358
359 def RunProcessSub(self, cs_part):
360 # type: (CommandSub) -> str
361 return ''
362
363 def PushRedirects(self, redirects, err_out):
364 # type: (List[RedirValue], List[int]) -> None
365 pass
366
367 def PopRedirects(self, num_redirects, err_out):
368 # type: (int, List[int]) -> None
369 pass
370
371 def PushProcessSub(self):
372 # type: () -> None
373 pass
374
375 def PopProcessSub(self, compound_st):
376 # type: (StatusArray) -> None
377 pass
378
379
380#
381# Abstract base classes
382#
383
384
385class _AssignBuiltin(object):
386 """Interface for assignment builtins."""
387
388 def __init__(self):
389 # type: () -> None
390 """Empty constructor for mycpp."""
391 pass
392
393 def Run(self, cmd_val):
394 # type: (cmd_value.Assign) -> int
395 raise NotImplementedError()
396
397
398class _Builtin(object):
399 """All builtins except 'command' obey this interface.
400
401 Assignment builtins use cmd_value.Assign; others use cmd_value.Argv.
402 """
403
404 def __init__(self):
405 # type: () -> None
406 """Empty constructor for mycpp."""
407 pass
408
409 def Run(self, cmd_val):
410 # type: (cmd_value.Argv) -> int
411 raise NotImplementedError()
412
413
414class _Callable(object):
415 """Interface for functions in the runtime."""
416
417 def __init__(self):
418 # type: () -> None
419 """Empty constructor for mycpp."""
420 pass
421
422 def Call(self, args):
423 # type: (typed_args.Reader) -> value_t
424 raise NotImplementedError()
425
426
427class ctx_MaybePure(object):
428 """Enforce purity of the shell interpreter
429
430 Use this for:
431
432 --eval-pure
433 func - pure functions
434 eval() evalToDict() - builtin pure functions, not methods
435 """
436
437 def __init__(
438 self,
439 pure_ex, # type: Optional[_Executor]
440 cmd_ev, # type: cmd_eval.CommandEvaluator
441 ):
442 # type: (...) -> None
443 self.pure_ex = pure_ex
444 if not pure_ex:
445 return # do nothing
446
447 word_ev = cmd_ev.word_ev
448 expr_ev = cmd_ev.expr_ev
449
450 # Save the Shell Executor
451 self.saved = cmd_ev.shell_ex
452 assert self.saved is word_ev.shell_ex
453 assert self.saved is expr_ev.shell_ex
454
455 # Patch evaluators to use the Pure Executor
456 cmd_ev.shell_ex = pure_ex
457 word_ev.shell_ex = pure_ex
458 expr_ev.shell_ex = pure_ex
459
460 self.cmd_ev = cmd_ev
461
462 def __enter__(self):
463 # type: () -> None
464 pass
465
466 def __exit__(self, type, value, traceback):
467 # type: (Any, Any, Any) -> None
468 if not self.pure_ex:
469 return
470
471 # Unpatch the evaluators
472 self.cmd_ev.shell_ex = self.saved
473 self.cmd_ev.word_ev.shell_ex = self.saved
474 self.cmd_ev.expr_ev.shell_ex = self.saved
475
476
477class ctx_Redirect(object):
478 """For closing files.
479
480 This is asymmetric because if PushRedirects fails, then we don't execute
481 the command at all.
482
483 Example:
484 { seq 3 > foo.txt; echo 4; } > bar.txt
485 """
486
487 def __init__(self, shell_ex, num_redirects, err_out):
488 # type: (_Executor, int, List[int]) -> None
489 self.shell_ex = shell_ex
490 self.num_redirects = num_redirects
491 self.err_out = err_out
492
493 def __enter__(self):
494 # type: () -> None
495 pass
496
497 def __exit__(self, type, value, traceback):
498 # type: (Any, Any, Any) -> None
499 self.shell_ex.PopRedirects(self.num_redirects, self.err_out)
500
501
502class ctx_ProcessSub(object):
503 """For waiting on processes started during word evaluation.
504
505 Example:
506 diff <(seq 3) <(seq 4) > >(tac)
507 """
508
509 def __init__(self, shell_ex, process_sub_status):
510 # type: (_Executor, StatusArray) -> None
511 shell_ex.PushProcessSub()
512 self.shell_ex = shell_ex
513 self.process_sub_status = process_sub_status
514
515 def __enter__(self):
516 # type: () -> None
517 pass
518
519 def __exit__(self, type, value, traceback):
520 # type: (Any, Any, Any) -> None
521
522 # Wait and return array to set _process_sub_status
523 self.shell_ex.PopProcessSub(self.process_sub_status)
524
525
526class ctx_FlushStdout(object):
527
528 def __init__(self, err_out):
529 # type: (List[error.IOError_OSError]) -> None
530 self.err_out = err_out
531
532 def __enter__(self):
533 # type: () -> None
534 pass
535
536 def __exit__(self, type, value, traceback):
537 # type: (Any, Any, Any) -> None
538
539 # Can't raise exception in destructor! So we append it to out param.
540 err = pyos.FlushStdout()
541 if err is not None:
542 self.err_out.append(err)