OILS / builtin / meta_oils.py View on Github | oilshell.org

692 lines, 395 significant
1#!/usr/bin/env python2
2"""
3meta_oils.py - Builtins that call back into the interpreter, or reflect on it.
4
5OSH builtins:
6 builtin command type
7 source eval
8
9YSH builtins:
10 invoke extern
11 use
12"""
13from __future__ import print_function
14
15from _devbuild.gen import arg_types
16from _devbuild.gen.runtime_asdl import cmd_value, CommandStatus
17from _devbuild.gen.syntax_asdl import source, loc, loc_t
18from core import alloc
19from core import dev
20from core import error
21from core import executor
22from core import main_loop
23from core import process
24from core.error import e_usage
25from core import pyutil # strerror
26from core import state
27from core import vm
28from data_lang import j8_lite
29from frontend import flag_util
30from frontend import consts
31from frontend import reader
32from mycpp.mylib import log, print_stderr
33from pylib import os_path
34from osh import cmd_eval
35
36import posix_ as posix
37from posix_ import X_OK # translated directly to C macro
38
39_ = log
40
41from typing import Dict, List, Tuple, Optional, TYPE_CHECKING
42if TYPE_CHECKING:
43 from frontend import args
44 from frontend.parse_lib import ParseContext
45 from core import optview
46 from display import ui
47 from mycpp import mylib
48 from osh.cmd_eval import CommandEvaluator
49 from osh import cmd_parse
50
51
52class Eval(vm._Builtin):
53
54 def __init__(
55 self,
56 parse_ctx, # type: ParseContext
57 exec_opts, # type: optview.Exec
58 cmd_ev, # type: CommandEvaluator
59 tracer, # type: dev.Tracer
60 errfmt, # type: ui.ErrorFormatter
61 mem, # type: state.Mem
62 ):
63 # type: (...) -> None
64 self.parse_ctx = parse_ctx
65 self.arena = parse_ctx.arena
66 self.exec_opts = exec_opts
67 self.cmd_ev = cmd_ev
68 self.tracer = tracer
69 self.errfmt = errfmt
70 self.mem = mem
71
72 def Run(self, cmd_val):
73 # type: (cmd_value.Argv) -> int
74
75 # There are no flags, but we need it to respect --
76 _, arg_r = flag_util.ParseCmdVal('eval', cmd_val)
77
78 if self.exec_opts.simple_eval_builtin():
79 code_str, eval_loc = arg_r.ReadRequired2('requires code string')
80 if not arg_r.AtEnd():
81 e_usage('requires exactly 1 argument', loc.Missing)
82 else:
83 code_str = ' '.join(arg_r.Rest())
84 # code_str could be EMPTY, so just use the first one
85 eval_loc = cmd_val.arg_locs[0]
86
87 line_reader = reader.StringLineReader(code_str, self.arena)
88 c_parser = self.parse_ctx.MakeOshParser(line_reader)
89
90 src = source.Dynamic('eval arg', eval_loc)
91 with dev.ctx_Tracer(self.tracer, 'eval', None):
92 with alloc.ctx_SourceCode(self.arena, src):
93 return main_loop.Batch(self.cmd_ev,
94 c_parser,
95 self.errfmt,
96 cmd_flags=cmd_eval.RaiseControlFlow)
97
98
99class ShellFile(vm._Builtin):
100 """
101 These share code:
102 - 'source' builtin for OSH
103 - 'use' builtin for YSH
104 """
105
106 def __init__(
107 self,
108 parse_ctx, # type: ParseContext
109 search_path, # type: state.SearchPath
110 cmd_ev, # type: CommandEvaluator
111 fd_state, # type: process.FdState
112 tracer, # type: dev.Tracer
113 errfmt, # type: ui.ErrorFormatter
114 loader, # type: pyutil._ResourceLoader
115 ysh_use=False, # type: bool
116 ):
117 # type: (...) -> None
118 self.parse_ctx = parse_ctx
119 self.arena = parse_ctx.arena
120 self.search_path = search_path
121 self.cmd_ev = cmd_ev
122 self.fd_state = fd_state
123 self.tracer = tracer
124 self.errfmt = errfmt
125 self.loader = loader
126 self.ysh_use = ysh_use
127
128 self.builtin_name = 'use' if ysh_use else 'source'
129 self.mem = cmd_ev.mem
130
131 def Run(self, cmd_val):
132 # type: (cmd_value.Argv) -> int
133 if self.ysh_use:
134 return self._Use(cmd_val)
135 else:
136 return self._Source(cmd_val)
137
138 def _LoadBuiltinFile(self, builtin_path, blame_loc):
139 # type: (str, loc_t) -> Tuple[str, cmd_parse.CommandParser]
140 try:
141 load_path = os_path.join("stdlib", builtin_path)
142 contents = self.loader.Get(load_path)
143 except (IOError, OSError):
144 self.errfmt.Print_('%r failed: No builtin file %r' %
145 (self.builtin_name, load_path),
146 blame_loc=blame_loc)
147 return None, None # error
148
149 line_reader = reader.StringLineReader(contents, self.arena)
150 c_parser = self.parse_ctx.MakeOshParser(line_reader)
151 return load_path, c_parser
152
153 def _LoadDiskFile(self, fs_path, blame_loc):
154 # type: (str, loc_t) -> Tuple[mylib.LineReader, cmd_parse.CommandParser]
155 try:
156 # Shell can't use descriptors 3-9
157 f = self.fd_state.Open(fs_path)
158 except (IOError, OSError) as e:
159 self.errfmt.Print_(
160 '%s %r failed: %s' %
161 (self.builtin_name, fs_path, pyutil.strerror(e)),
162 blame_loc=blame_loc)
163 return None, None
164
165 line_reader = reader.FileLineReader(f, self.arena)
166 c_parser = self.parse_ctx.MakeOshParser(line_reader)
167 return f, c_parser
168
169 def _Use(self, cmd_val):
170 # type: (cmd_value.Argv) -> int
171 """
172 Module system with all the power of Python, but still a proc
173
174 use util.ysh # util is a value.Obj
175
176 # Importing a bunch of words
177 use dialect-ninja.ysh { all } # requires 'provide' in dialect-ninja
178 use dialect-github.ysh { all }
179
180 # This declares some names
181 use --extern grep sed
182
183 # Renaming
184 use util.ysh (&myutil)
185
186 # Ignore
187 use util.ysh (&_)
188
189 # Picking specifics
190 use util.ysh {
191 pick log die
192 pick foo (&myfoo)
193 }
194
195 # A long way to write this is:
196
197 use util.ysh
198 const log = util.log
199 const die = util.die
200 const myfoo = util.foo
201
202 Another way is:
203 for name in log die {
204 call setVar(name, util[name])
205
206 # value.Obj may not support [] though
207 # get(propView(util), name, null) is a long way of writing it
208 }
209
210 Other considerations:
211
212 - Statically parseable subset? For fine-grained static tree-shaking
213 - We're doing coarse dynamic tree-shaking first though
214
215 - if TYPE_CHECKING is an issue
216 - that can create circular dependencies, especially with gradual typing,
217 when you go dynamic to static (like Oils did)
218 - I guess you can have
219 - use --static parse_lib.ysh { pick ParseContext }
220 """
221 _, arg_r = flag_util.ParseCmdVal('use', cmd_val)
222 path_arg, path_loc = arg_r.ReadRequired2('requires a module path')
223 # TODO on usage:
224 # - typed arg is value.Place
225 # - block arg binds 'pick' and 'all'
226 # Although ALL these 3 mechanisms can be done with 'const' assignments.
227 # Hm.
228 arg_r.Done()
229
230 # I wonder if modules should be FROZEN value.Obj, not mutable?
231
232 # Duplicating logic below
233 if path_arg.startswith('///'):
234 builtin_path = path_arg[3:]
235 else:
236 builtin_path = None
237
238 if builtin_path is not None:
239 load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc)
240 if c_parser is None:
241 return 1 # error was already shown
242
243 # TODO: ctx_Module
244 return self._Exec(cmd_val, arg_r, load_path, c_parser)
245 else:
246 f, c_parser = self._LoadDiskFile(path_arg, path_loc)
247 if c_parser is None:
248 return 1 # error was already shown
249
250 # TODO: ctx_Module
251 with process.ctx_FileCloser(f):
252 return self._Exec(cmd_val, arg_r, path_arg, c_parser)
253
254 raise AssertionError()
255
256 def _Source(self, cmd_val):
257 # type: (cmd_value.Argv) -> int
258 attrs, arg_r = flag_util.ParseCmdVal('source', cmd_val)
259 arg = arg_types.source(attrs.attrs)
260
261 path_arg, path_loc = arg_r.ReadRequired2('requires a file path')
262
263 # Old:
264 # source --builtin two.sh # looks up stdlib/two.sh
265 # New:
266 # source $LIB_OSH/two.sh # looks up stdlib/osh/two.sh
267 # source ///osh/two.sh # looks up stdlib/osh/two.sh
268 builtin_path = None # type: Optional[str]
269 if arg.builtin:
270 builtin_path = path_arg
271 elif path_arg.startswith('///'):
272 builtin_path = path_arg[3:]
273
274 if builtin_path is not None:
275 load_path, c_parser = self._LoadBuiltinFile(builtin_path, path_loc)
276 if c_parser is None:
277 return 1 # error was already shown
278
279 return self._Exec(cmd_val, arg_r, load_path, c_parser)
280
281 else:
282 # 'source' respects $PATH
283 resolved = self.search_path.LookupOne(path_arg,
284 exec_required=False)
285 if resolved is None:
286 resolved = path_arg
287
288 f, c_parser = self._LoadDiskFile(resolved, path_loc)
289 if c_parser is None:
290 return 1 # error was already shown
291
292 with process.ctx_FileCloser(f):
293 return self._Exec(cmd_val, arg_r, path_arg, c_parser)
294
295 raise AssertionError()
296
297 def _Exec(self, cmd_val, arg_r, path, c_parser):
298 # type: (cmd_value.Argv, args.Reader, str, cmd_parse.CommandParser) -> int
299 call_loc = cmd_val.arg_locs[0]
300
301 # A sourced module CAN have a new arguments array, but it always shares
302 # the same variable scope as the caller. The caller could be at either a
303 # global or a local scope.
304
305 # TODO: I wonder if we compose the enter/exit methods more easily.
306
307 with dev.ctx_Tracer(self.tracer, 'source', cmd_val.argv):
308 source_argv = arg_r.Rest()
309 with state.ctx_Source(self.mem, path, source_argv):
310 with state.ctx_ThisDir(self.mem, path):
311 src = source.SourcedFile(path, call_loc)
312 with alloc.ctx_SourceCode(self.arena, src):
313 try:
314 status = main_loop.Batch(
315 self.cmd_ev,
316 c_parser,
317 self.errfmt,
318 cmd_flags=cmd_eval.RaiseControlFlow)
319 except vm.IntControlFlow as e:
320 if e.IsReturn():
321 status = e.StatusCode()
322 else:
323 raise
324
325 return status
326
327
328def _PrintFreeForm(row):
329 # type: (Tuple[str, str, Optional[str]]) -> None
330 name, kind, resolved = row
331
332 if kind == 'file':
333 what = resolved
334 elif kind == 'alias':
335 what = ('an alias for %s' %
336 j8_lite.EncodeString(resolved, unquoted_ok=True))
337 elif kind in ('proc', 'invokable'):
338 # Note: haynode should be an invokable
339 what = 'a YSH %s' % kind
340 else: # builtin, function, keyword
341 what = 'a shell %s' % kind
342
343 print('%s is %s' % (name, what))
344
345 # if kind == 'function':
346 # bash is the only shell that prints the function
347
348
349def _PrintEntry(arg, row):
350 # type: (arg_types.type, Tuple[str, str, Optional[str]]) -> None
351
352 _, kind, resolved = row
353 assert kind is not None
354
355 if arg.t: # short string
356 print(kind)
357
358 elif arg.p:
359 #log('%s %s %s', name, kind, resolved)
360 if kind == 'file':
361 print(resolved)
362
363 else: # free-form text
364 _PrintFreeForm(row)
365
366
367class Command(vm._Builtin):
368 """'command ls' suppresses function lookup."""
369
370 def __init__(
371 self,
372 shell_ex, # type: vm._Executor
373 funcs, # type: state.Procs
374 aliases, # type: Dict[str, str]
375 search_path, # type: state.SearchPath
376 ):
377 # type: (...) -> None
378 self.shell_ex = shell_ex
379 self.funcs = funcs
380 self.aliases = aliases
381 self.search_path = search_path
382
383 def Run(self, cmd_val):
384 # type: (cmd_value.Argv) -> int
385
386 # accept_typed_args=True because we invoke other builtins
387 attrs, arg_r = flag_util.ParseCmdVal('command',
388 cmd_val,
389 accept_typed_args=True)
390 arg = arg_types.command(attrs.attrs)
391
392 argv, locs = arg_r.Rest2()
393
394 if arg.v or arg.V:
395 status = 0
396 for argument in argv:
397 r = _ResolveName(argument, self.funcs, self.aliases,
398 self.search_path, False)
399 if len(r):
400 # command -v prints the name (-V is more detailed)
401 # Print it only once.
402 row = r[0]
403 name, _, _ = row
404 if arg.v:
405 print(name)
406 else:
407 _PrintFreeForm(row)
408 else:
409 # match bash behavior by printing to stderr
410 print_stderr('%s: not found' % argument)
411 status = 1 # nothing printed, but we fail
412
413 return status
414
415 cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd,
416 cmd_val.proc_args)
417
418 cmd_st = CommandStatus.CreateNull(alloc_lists=True)
419
420 # If we respected do_fork here instead of passing DO_FORK
421 # unconditionally, the case 'command date | wc -l' would take 2
422 # processes instead of 3. See test/syscall
423 run_flags = executor.NO_CALL_PROCS
424 if cmd_val.is_last_cmd:
425 run_flags |= executor.IS_LAST_CMD
426 if arg.p:
427 run_flags |= executor.USE_DEFAULT_PATH
428
429 return self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, run_flags)
430
431
432def _ShiftArgv(cmd_val):
433 # type: (cmd_value.Argv) -> cmd_value.Argv
434 return cmd_value.Argv(cmd_val.argv[1:], cmd_val.arg_locs[1:],
435 cmd_val.is_last_cmd, cmd_val.proc_args)
436
437
438class Builtin(vm._Builtin):
439
440 def __init__(self, shell_ex, errfmt):
441 # type: (vm._Executor, ui.ErrorFormatter) -> None
442 self.shell_ex = shell_ex
443 self.errfmt = errfmt
444
445 def Run(self, cmd_val):
446 # type: (cmd_value.Argv) -> int
447
448 if len(cmd_val.argv) == 1:
449 return 0 # this could be an error in strict mode?
450
451 name = cmd_val.argv[1]
452
453 # Run regular builtin or special builtin
454 to_run = consts.LookupNormalBuiltin(name)
455 if to_run == consts.NO_INDEX:
456 to_run = consts.LookupSpecialBuiltin(name)
457 if to_run == consts.NO_INDEX:
458 location = cmd_val.arg_locs[1]
459 if consts.LookupAssignBuiltin(name) != consts.NO_INDEX:
460 # NOTE: There's a similar restriction for 'command'
461 self.errfmt.Print_("Can't run assignment builtin recursively",
462 blame_loc=location)
463 else:
464 self.errfmt.Print_("%r isn't a shell builtin" % name,
465 blame_loc=location)
466 return 1
467
468 cmd_val2 = _ShiftArgv(cmd_val)
469 return self.shell_ex.RunBuiltin(to_run, cmd_val2)
470
471
472class RunProc(vm._Builtin):
473
474 def __init__(self, shell_ex, procs, errfmt):
475 # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None
476 self.shell_ex = shell_ex
477 self.procs = procs
478 self.errfmt = errfmt
479
480 def Run(self, cmd_val):
481 # type: (cmd_value.Argv) -> int
482 _, arg_r = flag_util.ParseCmdVal('runproc',
483 cmd_val,
484 accept_typed_args=True)
485 argv, locs = arg_r.Rest2()
486
487 if len(argv) == 0:
488 raise error.Usage('requires arguments', loc.Missing)
489
490 name = argv[0]
491 proc, _ = self.procs.GetInvokable(name)
492 if not proc:
493 # note: should runproc be invoke?
494 self.errfmt.PrintMessage('runproc: no invokable named %r' % name)
495 return 1
496
497 cmd_val2 = cmd_value.Argv(argv, locs, cmd_val.is_last_cmd,
498 cmd_val.proc_args)
499
500 cmd_st = CommandStatus.CreateNull(alloc_lists=True)
501 run_flags = executor.IS_LAST_CMD if cmd_val.is_last_cmd else 0
502 return self.shell_ex.RunSimpleCommand(cmd_val2, cmd_st, run_flags)
503
504
505class Invoke(vm._Builtin):
506 """
507 Introspection:
508
509 invoke - YSH introspection on first word
510 type --all - introspection on variables too?
511 - different than = type(x)
512
513 3 Coarsed-grained categories
514 - invoke --builtin aka builtin
515 - including special builtins
516 - invoke --proc-like aka runproc
517 - myproc (42)
518 - sh-func
519 - invokable-obj
520 - invoke --extern aka extern
521
522 Note: If you don't distinguish between proc, sh-func, and invokable-obj,
523 then 'runproc' suffices.
524
525 invoke --proc-like reads more nicely though, and it also combines.
526
527 invoke --builtin --extern # this is like 'command'
528
529 You can also negate:
530
531 invoke --no-proc-like --no-builtin --no-extern
532
533 - type -t also has 'keyword' and 'assign builtin'
534
535 With no args, print a table of what's available
536
537 invoke --builtin
538 invoke --builtin true
539 """
540
541 def __init__(self, shell_ex, procs, errfmt):
542 # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None
543 self.shell_ex = shell_ex
544 self.procs = procs
545 self.errfmt = errfmt
546
547 def Run(self, cmd_val):
548 # type: (cmd_value.Argv) -> int
549 _, arg_r = flag_util.ParseCmdVal('invoke',
550 cmd_val,
551 accept_typed_args=True)
552 #argv, locs = arg_r.Rest2()
553
554 print('TODO: invoke')
555 # TODO
556 return 0
557
558
559class Extern(vm._Builtin):
560
561 def __init__(self, shell_ex, procs, errfmt):
562 # type: (vm._Executor, state.Procs, ui.ErrorFormatter) -> None
563 self.shell_ex = shell_ex
564 self.procs = procs
565 self.errfmt = errfmt
566
567 def Run(self, cmd_val):
568 # type: (cmd_value.Argv) -> int
569 _, arg_r = flag_util.ParseCmdVal('extern',
570 cmd_val,
571 accept_typed_args=True)
572 #argv, locs = arg_r.Rest2()
573
574 print('TODO: extern')
575
576 return 0
577
578
579def _ResolveName(
580 name, # type: str
581 procs, # type: state.Procs
582 aliases, # type: Dict[str, str]
583 search_path, # type: state.SearchPath
584 do_all, # type: bool
585):
586 # type: (...) -> List[Tuple[str, str, Optional[str]]]
587 """
588 TODO: Can this be moved to pure YSH?
589
590 All of these could be in YSH:
591
592 type, type -t, type -a
593 pp proc
594
595 We would have primitive isShellFunc() and isInvokableObj() functions
596 """
597
598 # MyPy tuple type
599 no_str = None # type: Optional[str]
600
601 results = [] # type: List[Tuple[str, str, Optional[str]]]
602
603 if procs:
604 if procs.IsShellFunc(name):
605 results.append((name, 'function', no_str))
606
607 if procs.IsProc(name):
608 results.append((name, 'proc', no_str))
609 elif procs.IsInvokableObj(name): # can't be both proc and obj
610 results.append((name, 'invokable', no_str))
611
612 if name in aliases:
613 results.append((name, 'alias', aliases[name]))
614
615 # See if it's a builtin
616 if consts.LookupNormalBuiltin(name) != 0:
617 results.append((name, 'builtin', no_str))
618 elif consts.LookupSpecialBuiltin(name) != 0:
619 results.append((name, 'builtin', no_str))
620 elif consts.LookupAssignBuiltin(name) != 0:
621 results.append((name, 'builtin', no_str))
622
623 # See if it's a keyword
624 if consts.IsControlFlow(name): # continue, etc.
625 results.append((name, 'keyword', no_str))
626 elif consts.IsKeyword(name):
627 results.append((name, 'keyword', no_str))
628
629 # See if it's external
630 for path in search_path.LookupReflect(name, do_all):
631 if posix.access(path, X_OK):
632 results.append((name, 'file', path))
633
634 return results
635
636
637class Type(vm._Builtin):
638
639 def __init__(
640 self,
641 funcs, # type: state.Procs
642 aliases, # type: Dict[str, str]
643 search_path, # type: state.SearchPath
644 errfmt, # type: ui.ErrorFormatter
645 ):
646 # type: (...) -> None
647 self.funcs = funcs
648 self.aliases = aliases
649 self.search_path = search_path
650 self.errfmt = errfmt
651
652 def Run(self, cmd_val):
653 # type: (cmd_value.Argv) -> int
654 attrs, arg_r = flag_util.ParseCmdVal('type', cmd_val)
655 arg = arg_types.type(attrs.attrs)
656
657 if arg.f: # suppress function lookup
658 funcs = None # type: state.Procs
659 else:
660 funcs = self.funcs
661
662 status = 0
663 names = arg_r.Rest()
664
665 if arg.P: # -P should forces PATH search, regardless of builtin/alias/function/etc.
666 for name in names:
667 paths = self.search_path.LookupReflect(name, arg.a)
668 if len(paths):
669 for path in paths:
670 print(path)
671 else:
672 status = 1
673 return status
674
675 for argument in names:
676 r = _ResolveName(argument, funcs, self.aliases, self.search_path,
677 arg.a)
678 if arg.a:
679 for row in r:
680 _PrintEntry(arg, row)
681 else:
682 if len(r): # Just print the first one
683 _PrintEntry(arg, r[0])
684
685 # Error case
686 if len(r) == 0:
687 if not arg.t: # 'type -t' is silent in this case
688 # match bash behavior by printing to stderr
689 print_stderr('%s: not found' % argument)
690 status = 1 # nothing printed, but we fail
691
692 return status