OILS / core / main_loop.py View on Github | oils.pub

513 lines, 287 significant
1"""main_loop.py.
2
3Variants:
4 main_loop.Interactive() calls ParseInteractiveLine() and ExecuteAndCatch()
5 main_loop.Batch() calls ParseLogicalLine() and ExecuteAndCatch()
6 main_loop.Headless() calls Batch() like eval and source.
7 We want 'echo 1\necho 2\n' to work, so we
8 don't bother with "the PS2 problem".
9 main_loop.ParseWholeFile() calls ParseLogicalLine(). Used by osh -n.
10"""
11from __future__ import print_function
12
13from _devbuild.gen import arg_types
14from _devbuild.gen.syntax_asdl import (command, command_t, parse_result,
15 parse_result_e, source)
16from core import alloc
17from core import error
18from core import process
19from core import pyos
20from core import state
21from core import util
22from display import ui
23from frontend import reader
24from osh import cmd_eval
25from mycpp import iolib
26from mycpp import mylib
27from mycpp.mylib import log, print_stderr, probe, tagswitch
28
29import fanos
30import posix_ as posix
31
32from typing import cast, Any, List, Tuple, TYPE_CHECKING
33if TYPE_CHECKING:
34 from core.comp_ui import _IDisplay
35 from core import process
36 from frontend import parse_lib
37 from osh import cmd_parse
38 from osh import cmd_eval
39 from osh.prompt import UserPlugin
40
41_ = log
42
43
44class ctx_Descriptors(object):
45 """Save and restore descriptor state for the headless EVAL command."""
46
47 def __init__(self, fds):
48 # type: (List[int]) -> None
49
50 self.saved0 = process.SaveFd(0)
51 self.saved1 = process.SaveFd(1)
52 self.saved2 = process.SaveFd(2)
53
54 #ShowDescriptorState('BEFORE')
55 posix.dup2(fds[0], 0)
56 posix.dup2(fds[1], 1)
57 posix.dup2(fds[2], 2)
58
59 self.fds = fds
60
61 def __enter__(self):
62 # type: () -> None
63 pass
64
65 def __exit__(self, type, value, traceback):
66 # type: (Any, Any, Any) -> None
67
68 # Restore
69 posix.dup2(self.saved0, 0)
70 posix.dup2(self.saved1, 1)
71 posix.dup2(self.saved2, 2)
72
73 # Restoration done, so close
74 posix.close(self.saved0)
75 posix.close(self.saved1)
76 posix.close(self.saved2)
77
78 # And close descriptors we were passed
79 posix.close(self.fds[0])
80 posix.close(self.fds[1])
81 posix.close(self.fds[2])
82
83
84def fanos_log(msg):
85 # type: (str) -> None
86 print_stderr('[FANOS] %s' % msg)
87
88
89def ShowDescriptorState(label):
90 # type: (str) -> None
91 if mylib.PYTHON:
92 import os # Our posix fork doesn't have os.system
93 import time
94 time.sleep(0.01) # prevent interleaving
95
96 pid = posix.getpid()
97 print_stderr(label + ' (PID %d)' % pid)
98
99 os.system('ls -l /proc/%d/fd >&2' % pid)
100
101 time.sleep(0.01) # prevent interleaving
102
103
104class Headless(object):
105 """Main loop for headless mode."""
106
107 def __init__(self, cmd_ev, parse_ctx, errfmt):
108 # type: (cmd_eval.CommandEvaluator, parse_lib.ParseContext, ui.ErrorFormatter) -> None
109 self.cmd_ev = cmd_ev
110 self.parse_ctx = parse_ctx
111 self.errfmt = errfmt
112
113 def Loop(self):
114 # type: () -> int
115 try:
116 return self._Loop()
117 except ValueError as e:
118 fanos.send(1, 'ERROR %s' % e)
119 return 1
120
121 def EVAL(self, arg):
122 # type: (str) -> str
123
124 # This logic is similar to the 'eval' builtin in osh/builtin_meta.
125
126 # Note: we're not using the InteractiveLineReader, so there's no history
127 # expansion. It would be nice if there was a way for the client to use
128 # that.
129 line_reader = reader.StringLineReader(arg, self.parse_ctx.arena)
130 c_parser = self.parse_ctx.MakeOshParser(line_reader)
131
132 # Status is unused; $_ can be queried by the headless client
133 unused_status = Batch(self.cmd_ev, c_parser, self.errfmt, 0)
134
135 return '' # result is always 'OK ' since there was no protocol error
136
137 def _Loop(self):
138 # type: () -> int
139 fanos_log(
140 'Connect stdin and stdout to one end of socketpair() and send control messages. osh writes debug messages (like this one) to stderr.'
141 )
142
143 fd_out = [] # type: List[int]
144 while True:
145 try:
146 blob = fanos.recv(0, fd_out)
147 except ValueError as e:
148 fanos_log('protocol error: %s' % e)
149 raise # higher level handles it
150
151 if blob is None:
152 fanos_log('EOF received')
153 break
154
155 fanos_log('received blob %r' % blob)
156 if ' ' in blob:
157 bs = blob.split(' ', 1)
158 command = bs[0]
159 arg = bs[1]
160 else:
161 command = blob
162 arg = ''
163
164 if command == 'GETPID':
165 reply = str(posix.getpid())
166
167 elif command == 'EVAL':
168 #fanos_log('arg %r', arg)
169
170 if len(fd_out) != 3:
171 raise ValueError('Expected 3 file descriptors')
172
173 for fd in fd_out:
174 fanos_log('received descriptor %d' % fd)
175
176 with ctx_Descriptors(fd_out):
177 reply = self.EVAL(arg)
178
179 #ShowDescriptorState('RESTORED')
180
181 # Note: lang == 'osh' or lang == 'ysh' puts this in different modes.
182 # Do we also need 'complete --osh' and 'complete --ysh' ?
183 elif command == 'PARSE':
184 # Just parse
185 reply = 'TODO:PARSE'
186
187 else:
188 fanos_log('Invalid command %r' % command)
189 raise ValueError('Invalid command %r' % command)
190
191 fanos.send(1, b'OK %s' % reply)
192 del fd_out[:] # reset for next iteration
193
194 return 0
195
196
197def _UpdateTerminalSize(mem, readline):
198 # type: (state.Mem, Any) -> None
199 """Query the terminal for its current size and update COLUMNS, LINES,
200 and readline's internal screen width.
201
202 Analogous to bash's sh_set_lines_and_columns().
203 """
204 rows, cols = pyos.GetTerminalSize()
205 if cols > 0:
206 state.SetGlobalString(mem, 'COLUMNS', str(cols))
207 if rows > 0:
208 state.SetGlobalString(mem, 'LINES', str(rows))
209 # Also update readline's internal idea of the terminal size so that
210 # line editing / wrapping works correctly.
211 if readline is not None:
212 readline.resize_terminal()
213
214
215def Interactive(
216 flag, # type: arg_types.main
217 cmd_ev, # type: cmd_eval.CommandEvaluator
218 c_parser, # type: cmd_parse.CommandParser
219 display, # type: _IDisplay
220 prompt_plugin, # type: UserPlugin
221 waiter, # type: process.Waiter
222 errfmt, # type: ui.ErrorFormatter
223 readline=None, # type: Any
224):
225 # type: (...) -> int
226 status = 0
227 done = False
228 while not done:
229 mylib.MaybeCollect() # manual GC point
230
231 # Update COLUMNS and LINES from the terminal every iteration.
232 # This keeps $COLUMNS/$LINES in sync after SIGWINCH, analogous to
233 # bash's sh_set_lines_and_columns() and checkwinsize.
234 _UpdateTerminalSize(cmd_ev.mem, readline)
235
236 # - This loop has a an odd structure because we want to do cleanup
237 # after every 'break'. (The ones without 'done = True' were
238 # 'continue')
239 # - display.EraseLines() needs to be called BEFORE displaying anything, so
240 # it appears in all branches.
241
242 while True: # ONLY EXECUTES ONCE
243 quit = False
244 prompt_plugin.Run()
245 try:
246 # may raise HistoryError or ParseError
247 result = c_parser.ParseInteractiveLine()
248 UP_result = result
249 with tagswitch(result) as case:
250 if case(parse_result_e.EmptyLine):
251 display.EraseLines()
252 # POSIX shell behavior: waitpid(-1) and show job "Done"
253 # messages
254 waiter.PollForEvents()
255 quit = True
256 elif case(parse_result_e.Eof):
257 display.EraseLines()
258 done = True
259 quit = True
260 elif case(parse_result_e.Node):
261 result = cast(parse_result.Node, UP_result)
262 node = result.cmd
263 else:
264 raise AssertionError()
265
266 except util.HistoryError as e: # e.g. expansion failed
267 # Where this happens:
268 # for i in 1 2 3; do
269 # !invalid
270 # done
271 display.EraseLines()
272 print(e.UserErrorString())
273 quit = True
274 except error.Parse as e:
275 display.EraseLines()
276 errfmt.PrettyPrintError(e)
277 status = 2
278 cmd_ev.mem.SetLastStatus(status)
279 quit = True
280 except KeyboardInterrupt: # thrown by InteractiveLineReader._GetLine()
281 # TODO: We probably want to change terminal settings so ^C is printed.
282 # For now, just print a newline.
283 #
284 # WITHOUT GNU readline, the ^C is printed. So we need to make
285 # the 2 cases consistent.
286 print('')
287
288 if 0:
289 from core import pyos
290 pyos.FlushStdout()
291
292 display.EraseLines()
293 quit = True
294
295 if quit:
296 break
297
298 display.EraseLines() # Clear candidates right before executing
299
300 # to debug the slightly different interactive prasing
301 if cmd_ev.exec_opts.noexec():
302 ui.PrintAst(node, flag)
303 break
304
305 try:
306 is_return, _ = cmd_ev.ExecuteAndCatch(node, 0)
307 except KeyboardInterrupt: # issue 467, Ctrl-C during $(sleep 1)
308 is_return = False
309 display.EraseLines()
310
311 # http://www.tldp.org/LDP/abs/html/exitcodes.html
312 # bash gives 130, dash gives 0, zsh gives 1.
313 status = 130 # 128 + 2
314
315 cmd_ev.mem.SetLastStatus(status)
316 break
317
318 status = cmd_ev.LastStatus()
319
320 waiter.PollForEvents()
321
322 if is_return:
323 done = True
324 break
325
326 break # QUIT LOOP after one iteration.
327
328 # After every "logical line", no lines will be referenced by the Arena.
329 # Tokens in the LST still point to many lines, but lines with only comment
330 # or whitespace won't be reachable, so the GC will free them.
331 c_parser.arena.DiscardLines()
332
333 cmd_ev.RunPendingTraps() # Run trap handlers even if we get just ENTER
334
335 # Cleanup after every command (or failed command).
336
337 # Reset internal newline state.
338 c_parser.Reset()
339 c_parser.ResetInputObjects()
340
341 display.Reset() # clears dupes and number of lines last displayed
342
343 # TODO: Replace this with a shell hook? with 'trap', or it could be just
344 # like command_not_found. The hook can be 'echo $?' or something more
345 # complicated, i.e. with timestamps.
346 if flag.print_status:
347 print('STATUS\t%r' % status)
348
349 return status
350
351
352def Batch(
353 cmd_ev, # type: cmd_eval.CommandEvaluator
354 c_parser, # type: cmd_parse.CommandParser
355 errfmt, # type: ui.ErrorFormatter
356 cmd_flags=0, # type: int
357):
358 # type: (...) -> int
359 """
360 source, eval, etc. treat parse errors as error code 2. But the --eval flag does not.
361 """
362 was_parsed, status = Batch2(cmd_ev, c_parser, errfmt, cmd_flags=cmd_flags)
363 if not was_parsed:
364 return 2
365 return status
366
367
368def Batch2(
369 cmd_ev, # type: cmd_eval.CommandEvaluator
370 c_parser, # type: cmd_parse.CommandParser
371 errfmt, # type: ui.ErrorFormatter
372 cmd_flags=0, # type: int
373):
374 # type: (...) -> Tuple[bool, int]
375 """Loop for batch execution.
376
377 Returns:
378 int status, e.g. 2 on parse error
379
380 Can this be combined with interactive loop? Differences:
381
382 - Handling of parse errors.
383 - Have to detect here docs at the end?
384
385 Not a problem:
386 - Get rid of --print-status and --show-ast for now
387 - Get rid of EOF difference
388
389 TODO:
390 - Do source / eval need this?
391 - 'source' needs to parse incrementally so that aliases are respected
392 - I doubt 'eval' does! You can test it.
393 - In contrast, 'trap' should parse up front?
394 - What about $() ?
395 """
396 was_parsed = True
397 status = 0
398 while True:
399 probe('main_loop', 'Batch_parse_enter')
400 try:
401 node = c_parser.ParseLogicalLine() # can raise ParseError
402 if node is None: # EOF
403 c_parser.CheckForPendingHereDocs() # can raise ParseError
404 break
405 except error.Parse as e:
406 errfmt.PrettyPrintError(e)
407 was_parsed = False
408 status = -1 # invalid value
409 break
410
411 # After every "logical line", no lines will be referenced by the Arena.
412 # Tokens in the LST still point to many lines, but lines with only comment
413 # or whitespace won't be reachable, so the GC will free them.
414 c_parser.arena.DiscardLines()
415
416 # Only optimize if we're on the last line like -c "echo hi" etc.
417 if (cmd_flags & cmd_eval.IsMainProgram and
418 c_parser.line_reader.LastLineHint()):
419 cmd_flags |= cmd_eval.OptimizeSubshells
420 if not cmd_ev.exec_opts.verbose_errexit():
421 cmd_flags |= cmd_eval.MarkLastCommands
422
423 probe('main_loop', 'Batch_parse_exit')
424
425 probe('main_loop', 'Batch_execute_enter')
426 # can't optimize this because we haven't seen the end yet
427 is_return, is_fatal = cmd_ev.ExecuteAndCatch(node, cmd_flags)
428 status = cmd_ev.LastStatus()
429 # e.g. 'return' in middle of script, or divide by zero
430 if is_return or is_fatal:
431 break
432 probe('main_loop', 'Batch_execute_exit')
433
434 probe('main_loop', 'Batch_collect_enter')
435 mylib.MaybeCollect() # manual GC point
436 probe('main_loop', 'Batch_collect_exit')
437
438 return was_parsed, status
439
440
441def ParseWholeFile(c_parser):
442 # type: (cmd_parse.CommandParser) -> command_t
443 """Parse an entire shell script.
444
445 This uses the same logic as Batch(). Used by:
446 - osh -n
447 - oshc translate
448 - Used by 'trap' to store code. But 'source' and 'eval' use Batch().
449
450 Note: it does NOT call DiscardLines
451 """
452 children = [] # type: List[command_t]
453 while True:
454 node = c_parser.ParseLogicalLine() # can raise ParseError
455 if node is None: # EOF
456 c_parser.CheckForPendingHereDocs() # can raise ParseError
457 break
458 children.append(node)
459
460 mylib.MaybeCollect() # manual GC point
461
462 if len(children) == 1:
463 return children[0]
464 else:
465 return command.CommandList(children)
466
467
468def EvalFile(
469 fs_path, # type: str
470 fd_state, # type: process.FdState
471 parse_ctx, # type: parse_lib.ParseContext
472 cmd_ev, # type: cmd_eval.CommandEvaluator
473 lang, # type: str
474):
475 # type: (...) -> Tuple[bool, int]
476 """Evaluate a disk file, for --eval --eval-pure
477
478 Copied and adapted from the 'source' builtin in builtin/meta_oils.py.
479
480 (Note that bind -x has to eval from a string, like Eval)
481
482 Raises:
483 util.HardExit
484 Returns:
485 ok: whether processing should continue
486 """
487 try:
488 f = fd_state.Open(fs_path)
489 except (IOError, OSError) as e:
490 print_stderr("%s: Couldn't open %r for --eval: %s" %
491 (lang, fs_path, posix.strerror(e.errno)))
492 return False, -1
493
494 line_reader = reader.FileLineReader(f, cmd_ev.arena)
495 c_parser = parse_ctx.MakeOshParser(line_reader)
496
497 # TODO:
498 # - Improve error locations
499 # - parse error should be fatal
500
501 with process.ctx_FileCloser(f):
502 with state.ctx_Eval(cmd_ev.mem, fs_path, None,
503 None): # set $0 to fs_path
504 with state.ctx_ThisDir(cmd_ev.mem, fs_path):
505 src = source.MainFile(fs_path)
506 with alloc.ctx_SourceCode(cmd_ev.arena, src):
507 # May raise util.HardExit
508 was_parsed, status = Batch2(cmd_ev, c_parser,
509 cmd_ev.errfmt)
510 if not was_parsed:
511 return False, -1
512
513 return True, status