core/state.py

OILS / core / state.py View on Github | oils.pub

2839 lines, 1448 significant

1	# Copyright 2016 Andy Chu. All rights reserved.
2	# Licensed under the Apache License, Version 2.0 (the "License");
3	# you may not use this file except in compliance with the License.
4	# You may obtain a copy of the License at
5	#
6	# http://www.apache.org/licenses/LICENSE-2.0
7	"""
8	state.py - Interpreter state
9	"""
10	from __future__ import print_function
11	import time as time_ # avoid name conflict
12
13	from _devbuild.gen.id_kind_asdl import Id
14	from _devbuild.gen.option_asdl import option_i
15	from _devbuild.gen.runtime_asdl import (error_code_e, scope_e, scope_t, Cell)
16	from _devbuild.gen.syntax_asdl import (loc, loc_t, Token, debug_frame,
17	debug_frame_e, debug_frame_t)
18	from _devbuild.gen.types_asdl import opt_group_i
19	from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj, sh_lvalue,
20	sh_lvalue_e, sh_lvalue_t, LeftName,
21	y_lvalue_e, regex_match, regex_match_e,
22	regex_match_t, RegexMatch)
23	from core import bash_impl
24	from core import error
25	from core.error import e_usage, e_die
26	from core import num
27	from core import optview
28	from display import ui
29	from core import util
30	from frontend import consts
31	from frontend import location
32	from frontend import match
33	from mycpp import mops
34	from mycpp import mylib
35	from mycpp.mylib import (log, print_stderr, str_switch, tagswitch, iteritems,
36	NewDict)
37	from pylib import os_path
38
39	from libc import HAVE_GLOB_PERIOD
40	import posix_ as posix
41
42	from typing import Tuple, List, Dict, Optional, Any, cast, TYPE_CHECKING
43
44	if TYPE_CHECKING:
45	from _devbuild.gen.option_asdl import option_t
46	from core import alloc
47	from osh import sh_expr_eval
48
49	_ = log
50
51	# flags for mem.SetValue()
52	SetReadOnly = 1 << 0
53	ClearReadOnly = 1 << 1
54	SetExport = 1 << 2
55	ClearExport = 1 << 3
56	SetNameref = 1 << 4
57	ClearNameref = 1 << 5
58
59
60	class ctx_Source(object):
61	"""For source builtin."""
62
63	def __init__(self, mem, source_name, argv):
64	# type: (Mem, str, List[str]) -> None
65	mem.PushSource(source_name, argv)
66	self.mem = mem
67	self.argv = argv
68
69	# Whenever we're sourcing, the 'is-main' builtin will return 1 (false)
70	self.to_restore = self.mem.is_main
71	self.mem.is_main = False
72
73	def __enter__(self):
74	# type: () -> None
75	pass
76
77	def __exit__(self, type, value, traceback):
78	# type: (Any, Any, Any) -> None
79	self.mem.PopSource(self.argv)
80
81	self.mem.is_main = self.to_restore
82
83
84	class ctx_DebugTrap(object):
85	"""For trap DEBUG."""
86
87	def __init__(self, mem):
88	# type: (Mem) -> None
89	mem.running_debug_trap = True
90	self.mem = mem
91
92	def __enter__(self):
93	# type: () -> None
94	pass
95
96	def __exit__(self, type, value, traceback):
97	# type: (Any, Any, Any) -> None
98	self.mem.running_debug_trap = False
99
100
101	class ctx_ErrTrap(object):
102	"""For trap ERR."""
103
104	def __init__(self, mem):
105	# type: (Mem) -> None
106	mem.running_err_trap = True
107	self.mem = mem
108
109	def __enter__(self):
110	# type: () -> None
111	pass
112
113	def __exit__(self, type, value, traceback):
114	# type: (Any, Any, Any) -> None
115	self.mem.running_err_trap = False
116
117
118	class ctx_Option(object):
119	"""Shopt --unset errexit { false }"""
120
121	def __init__(self, mutable_opts, opt_nums, b):
122	# type: (MutableOpts, List[int], bool) -> None
123	for opt_num in opt_nums:
124	mutable_opts.Push(opt_num, b)
125	if opt_num == option_i.errexit:
126	# it wasn't disabled
127	mutable_opts.errexit_disabled_tok.append(None)
128
129	self.mutable_opts = mutable_opts
130	self.opt_nums = opt_nums
131
132	def __enter__(self):
133	# type: () -> None
134	pass
135
136	def __exit__(self, type, value, traceback):
137	# type: (Any, Any, Any) -> None
138	for opt_num in self.opt_nums: # don't bother to do it in reverse order
139	if opt_num == option_i.errexit:
140	self.mutable_opts.errexit_disabled_tok.pop()
141	self.mutable_opts.Pop(opt_num)
142
143
144	class ctx_AssignBuiltin(object):
145	"""Local x=$(false) is disallowed."""
146
147	def __init__(self, mutable_opts):
148	# type: (MutableOpts) -> None
149	self.strict = False
150	if mutable_opts.Get(option_i.strict_errexit):
151	mutable_opts.Push(option_i._allow_command_sub, False)
152	mutable_opts.Push(option_i._allow_process_sub, False)
153	self.strict = True
154
155	self.mutable_opts = mutable_opts
156
157	def __enter__(self):
158	# type: () -> None
159	pass
160
161	def __exit__(self, type, value, traceback):
162	# type: (Any, Any, Any) -> None
163	if self.strict:
164	self.mutable_opts.Pop(option_i._allow_command_sub)
165	self.mutable_opts.Pop(option_i._allow_process_sub)
166
167
168	class ctx_YshExpr(object):
169	"""Command sub must fail in 'mystring' ++ $(false)"""
170
171	def __init__(self, mutable_opts):
172	# type: (MutableOpts) -> None
173
174	# Similar to $LIB_OSH/bash-strict.sh
175
176	# TODO: consider errexit:all group, or even ysh:all
177	# It would be nice if this were more efficient
178	mutable_opts.Push(option_i.command_sub_errexit, True)
179	mutable_opts.Push(option_i.errexit, True)
180	mutable_opts.Push(option_i.pipefail, True)
181	mutable_opts.Push(option_i.inherit_errexit, True)
182	mutable_opts.Push(option_i.strict_errexit, True)
183
184	# What about nounset? This has a similar pitfall -- it's not running
185	# like YSH.
186	# e.g. var x = $(echo $zz)
187
188	self.mutable_opts = mutable_opts
189
190	def __enter__(self):
191	# type: () -> None
192	pass
193
194	def __exit__(self, type, value, traceback):
195	# type: (Any, Any, Any) -> None
196	self.mutable_opts.Pop(option_i.command_sub_errexit)
197	self.mutable_opts.Pop(option_i.errexit)
198	self.mutable_opts.Pop(option_i.pipefail)
199	self.mutable_opts.Pop(option_i.inherit_errexit)
200	self.mutable_opts.Pop(option_i.strict_errexit)
201
202
203	class ctx_ErrExit(object):
204	"""Manages the errexit setting.
205
206	- The user can change it with builtin 'set' at any point in the code.
207	- These constructs implicitly disable 'errexit':
208	- if / while / until conditions
209	- ! (part of pipeline)
210	- && \|\|
211	"""
212
213	def __init__(self, mutable_opts, b, disabled_tok):
214	# type: (MutableOpts, bool, Optional[Token]) -> None
215
216	# If we're disabling it, we need a span ID. If not, then we should NOT
217	# have one.
218	assert b == (disabled_tok is None)
219
220	mutable_opts.Push(option_i.errexit, b)
221	mutable_opts.errexit_disabled_tok.append(disabled_tok)
222
223	self.strict = False
224	if mutable_opts.Get(option_i.strict_errexit):
225	mutable_opts.Push(option_i._allow_command_sub, False)
226	mutable_opts.Push(option_i._allow_process_sub, False)
227	self.strict = True
228
229	self.mutable_opts = mutable_opts
230
231	def __enter__(self):
232	# type: () -> None
233	pass
234
235	def __exit__(self, type, value, traceback):
236	# type: (Any, Any, Any) -> None
237	self.mutable_opts.errexit_disabled_tok.pop()
238	self.mutable_opts.Pop(option_i.errexit)
239
240	if self.strict:
241	self.mutable_opts.Pop(option_i._allow_command_sub)
242	self.mutable_opts.Pop(option_i._allow_process_sub)
243
244
245	class OptHook(object):
246	"""Interface for option hooks."""
247
248	def __init__(self):
249	# type: () -> None
250	"""Empty constructor for mycpp."""
251	pass
252
253	def OnChange(self, opt0_array, opt_name, b):
254	# type: (List[bool], str, bool) -> bool
255	"""This method is called whenever an option is changed.
256
257	Returns success or failure.
258	"""
259	return True
260
261
262	def InitOpts():
263	# type: () -> List[bool]
264
265	opt0_array = [False] * option_i.ARRAY_SIZE
266	for opt_num in consts.DEFAULT_TRUE:
267	opt0_array[opt_num] = True
268	return opt0_array
269
270
271	def MakeOpts(
272	mem, # type: Mem
273	environ, # type: Dict[str, str]
274	opt_hook, # type: OptHook
275	):
276	# type: (...) -> Tuple[optview.Parse, optview.Exec, MutableOpts]
277
278	# Unusual representation: opt0_array + opt_stacks. For two features:
279	#
280	# - POSIX errexit disable semantics
281	# - YSH shopt --set nullglob { ... }
282	#
283	# We could do it with a single List of stacks. But because shopt --set
284	# random_option { ... } is very uncommon, we optimize and store the ZERO
285	# element of the stack in a flat array opt0_array (default False), and then
286	# the rest in opt_stacks, where the value could be None. By allowing the
287	# None value, we save ~50 or so list objects in the common case.
288
289	opt0_array = InitOpts()
290	# Overrides, including errexit
291	no_stack = None # type: List[bool] # for mycpp
292	opt_stacks = [no_stack] * option_i.ARRAY_SIZE # type: List[List[bool]]
293
294	parse_opts = optview.Parse(opt0_array, opt_stacks)
295	exec_opts = optview.Exec(opt0_array, opt_stacks)
296	mutable_opts = MutableOpts(mem, environ, opt0_array, opt_stacks, opt_hook)
297
298	return parse_opts, exec_opts, mutable_opts
299
300
301	def _SetGroup(opt0_array, opt_nums, b):
302	# type: (List[bool], List[int], bool) -> None
303	for opt_num in opt_nums:
304	b2 = not b if opt_num in consts.DEFAULT_TRUE else b
305	opt0_array[opt_num] = b2
306
307
308	def MakeYshParseOpts():
309	# type: () -> optview.Parse
310	opt0_array = InitOpts()
311	_SetGroup(opt0_array, consts.YSH_ALL, True)
312
313	no_stack = None # type: List[bool]
314	opt_stacks = [no_stack] * option_i.ARRAY_SIZE # type: List[List[bool]]
315
316	parse_opts = optview.Parse(opt0_array, opt_stacks)
317	return parse_opts
318
319
320	def _AnyOptionNum(opt_name, ignore_shopt_not_impl):
321	# type: (str, bool) -> option_t
322	opt_num = consts.OptionNum(opt_name)
323	if opt_num == 0:
324	if ignore_shopt_not_impl:
325	opt_num = consts.UnimplOptionNum(opt_name)
326	if opt_num == 0:
327	e_usage('got invalid option %r' % opt_name, loc.Missing)
328
329	# Note: we relaxed this for YSH so we can do 'shopt --unset errexit' consistently
330	#if opt_num not in consts.SHOPT_OPTION_NUMS:
331	# e_usage("doesn't own option %r (try 'set')" % opt_name)
332
333	return opt_num
334
335
336	def _SetOptionNum(opt_name):
337	# type: (str) -> option_t
338	opt_num = consts.OptionNum(opt_name)
339	if opt_num == 0:
340	e_usage('got invalid option %r' % opt_name, loc.Missing)
341
342	if opt_num not in consts.SET_OPTION_NUMS:
343	e_usage("invalid option %r (try shopt)" % opt_name, loc.Missing)
344
345	return opt_num
346
347
348	def _MaybeWarnDotglob():
349	# type: () -> None
350	if HAVE_GLOB_PERIOD == 0:
351	# GNU libc and musl libc have GLOB_PERIOD, but Android doesn't
352	print_stderr(
353	"osh warning: GLOB_PERIOD wasn't found in libc, so 'shopt -s dotglob' won't work"
354	)
355
356
357	class MutableOpts(object):
358
359	def __init__(self, mem, environ, opt0_array, opt_stacks, opt_hook):
360	# type: (Mem, Dict[str, str], List[bool], List[List[bool]], OptHook) -> None
361	self.mem = mem
362	self.environ = environ
363	self.opt0_array = opt0_array
364	self.opt_stacks = opt_stacks
365	self.errexit_disabled_tok = [] # type: List[Token]
366
367	# Used for 'set -o vi/emacs'
368	self.opt_hook = opt_hook
369
370	def Init(self):
371	# type: () -> None
372
373	# This comes after all the 'set' options.
374	shellopts = self.mem.GetValue('SHELLOPTS')
375
376	# True in OSH, but not in YSH (no_init_globals)
377	if shellopts.tag() == value_e.Str:
378	s = cast(value.Str, shellopts).s
379	self._InitOptionsFromEnv(s)
380
381	def _InitOptionsFromEnv(self, shellopts):
382	# type: (str) -> None
383	# e.g. errexit:nounset:pipefail
384	lookup = shellopts.split(':')
385	for opt_num in consts.SET_OPTION_NUMS:
386	name = consts.OptionName(opt_num)
387	if name in lookup:
388	self._SetOldOption(name, True)
389
390	def Push(self, opt_num, b):
391	# type: (int, bool) -> None
392	if opt_num == option_i.dotglob:
393	_MaybeWarnDotglob()
394
395	overlay = self.opt_stacks[opt_num]
396	if overlay is None or len(overlay) == 0:
397	self.opt_stacks[opt_num] = [b] # Allocate a new list
398	else:
399	overlay.append(b)
400
401	def Pop(self, opt_num):
402	# type: (int) -> bool
403	overlay = self.opt_stacks[opt_num]
404	assert overlay is not None
405	return overlay.pop()
406
407	def PushDynamicScope(self, b):
408	# type: (bool) -> None
409	"""B: False if it's a proc, and True if it's a shell function."""
410	# If it's already disabled, keep it disabled
411	if not self.Get(option_i.dynamic_scope):
412	b = False
413	self.Push(option_i.dynamic_scope, b)
414
415	def PopDynamicScope(self):
416	# type: () -> None
417	self.Pop(option_i.dynamic_scope)
418
419	def Get(self, opt_num):
420	# type: (int) -> bool
421	# Like _Getter in core/optview.py
422	overlay = self.opt_stacks[opt_num]
423	if overlay is None or len(overlay) == 0:
424	return self.opt0_array[opt_num]
425	else:
426	return overlay[-1] # the top value
427
428	def _Set(self, opt_num, b):
429	# type: (int, bool) -> None
430	"""Used to disable errexit.
431
432	For bash compatibility in command sub.
433	"""
434	if opt_num == option_i.dotglob:
435	_MaybeWarnDotglob()
436
437	# Like _Getter in core/optview.py
438	overlay = self.opt_stacks[opt_num]
439	if overlay is None or len(overlay) == 0:
440	self.opt0_array[opt_num] = b
441	else:
442	overlay[-1] = b # The top value
443
444	def set_interactive(self):
445	# type: () -> None
446	self._Set(option_i.interactive, True)
447
448	def set_redefine_const(self):
449	# type: () -> None
450	"""For interactive shells."""
451	self._Set(option_i.redefine_const, True)
452
453	def set_redefine_source(self):
454	# type: () -> None
455	"""For interactive shells. For source-guard"""
456	self._Set(option_i.redefine_source, True)
457
458	def set_emacs(self):
459	# type: () -> None
460	self._Set(option_i.emacs, True)
461
462	def _SetArrayByNum(self, opt_num, b):
463	# type: (int, bool) -> None
464	if (opt_num in consts.PARSE_OPTION_NUMS and
465	not self.mem.ParsingChangesAllowed()):
466	e_die('Syntax options must be set at the top level '
467	'(outside any function)')
468
469	self._Set(opt_num, b)
470
471	def SetDeferredErrExit(self, b):
472	# type: (bool) -> None
473	"""Set the errexit flag, possibly deferring it.
474
475	Implements the unusual POSIX "defer" behavior. Callers: set -o
476	errexit, shopt -s ysh:all, ysh:upgrade
477	"""
478	#log('Set %s', b)
479
480	# Defer it until we pop by setting the BOTTOM OF THE STACK.
481	self.opt0_array[option_i.errexit] = b
482
483	def DisableErrExit(self):
484	# type: () -> None
485	"""Called by core/process.py to implement bash quirks."""
486	self._Set(option_i.errexit, False)
487
488	def ErrExitDisabledToken(self):
489	# type: () -> Optional[Token]
490	"""If errexit is disabled by POSIX rules, return Token for construct.
491
492	e.g. the Token for 'if' or '&&' etc.
493	"""
494	# Bug fix: The errexit disabling inherently follows a STACK DISCIPLINE.
495	# But we run trap handlers in the MAIN LOOP, which break this. So just
496	# declare that it's never disabled in a trap.
497	if self.Get(option_i._running_trap):
498	return None
499
500	if len(self.errexit_disabled_tok) == 0:
501	return None
502
503	return self.errexit_disabled_tok[-1]
504
505	def ErrExitIsDisabled(self):
506	# type: () -> bool
507	"""
508	Similar to ErrExitDisabledToken, for ERR trap
509	"""
510	if len(self.errexit_disabled_tok) == 0:
511	return False
512
513	return self.errexit_disabled_tok[-1] is not None
514
515	def _SetOldOption(self, opt_name, b):
516	# type: (str, bool) -> None
517	"""Private version for synchronizing from SHELLOPTS."""
518	assert '_' not in opt_name
519	assert opt_name in consts.SET_OPTION_NAMES
520
521	opt_num = consts.OptionNum(opt_name)
522	assert opt_num != 0, opt_name
523
524	if opt_num == option_i.errexit:
525	self.SetDeferredErrExit(b)
526	else:
527	if opt_num == option_i.verbose and b:
528	print_stderr('osh warning: set -o verbose not implemented')
529	self._SetArrayByNum(opt_num, b)
530
531	# note: may FAIL before we get here.
532
533	success = self.opt_hook.OnChange(self.opt0_array, opt_name, b)
534
535	def SetOldOption(self, opt_name, b):
536	# type: (str, bool) -> None
537	"""For set -o, set +o, or shopt -s/-u -o."""
538	unused = _SetOptionNum(opt_name) # validate it
539	self._SetOldOption(opt_name, b)
540
541	if not self.Get(option_i.no_init_globals):
542	UP_val = self.mem.GetValue('SHELLOPTS')
543	assert UP_val.tag() == value_e.Str, UP_val
544	val = cast(value.Str, UP_val)
545	shellopts = val.s
546
547	# Now check if SHELLOPTS needs to be updated. It may be exported.
548	#
549	# NOTE: It might be better to skip rewriting SEHLLOPTS in the common case
550	# where it is not used. We could do it lazily upon GET.
551
552	# Also, it would be slightly more efficient to update SHELLOPTS if
553	# settings were batched, Examples:
554	# - set -eu
555	# - shopt -s foo bar
556	if b:
557	if opt_name not in shellopts:
558	new_val = value.Str('%s:%s' % (shellopts, opt_name))
559	self.mem.InternalSetGlobal('SHELLOPTS', new_val)
560	else:
561	if opt_name in shellopts:
562	names = [n for n in shellopts.split(':') if n != opt_name]
563	new_val = value.Str(':'.join(names))
564	self.mem.InternalSetGlobal('SHELLOPTS', new_val)
565
566	def SetAnyOption(self, opt_name, b, ignore_shopt_not_impl=False):
567	# type: (str, bool, bool) -> None
568	"""For shopt -s/-u and sh -O/+O."""
569
570	# shopt -s ysh:all turns on all YSH options, which includes all strict
571	# options
572	opt_group = consts.OptionGroupNum(opt_name)
573	if opt_group == opt_group_i.YshUpgrade:
574	_SetGroup(self.opt0_array, consts.YSH_UPGRADE, b)
575	self.SetDeferredErrExit(b) # Special case
576	if b: # ENV dict
577	self.mem.MaybeInitEnvDict(self.environ)
578	return
579
580	if opt_group == opt_group_i.YshAll:
581	_SetGroup(self.opt0_array, consts.YSH_ALL, b)
582	self.SetDeferredErrExit(b) # Special case
583	if b: # ENV dict
584	self.mem.MaybeInitEnvDict(self.environ)
585	return
586
587	if opt_group == opt_group_i.StrictAll:
588	_SetGroup(self.opt0_array, consts.STRICT_ALL, b)
589	return
590
591	opt_num = _AnyOptionNum(opt_name, ignore_shopt_not_impl)
592
593	if opt_num == option_i.errexit:
594	self.SetDeferredErrExit(b)
595	return
596
597	self._SetArrayByNum(opt_num, b)
598
599
600	class _ArgFrame(object):
601	"""Stack frame for arguments array."""
602
603	def __init__(self, argv):
604	# type: (List[str]) -> None
605	self.argv = argv
606	self.num_shifted = 0
607
608	def __repr__(self):
609	# type: () -> str
610	return '<_ArgFrame %s %d at %x>' % (self.argv, self.num_shifted,
611	id(self))
612
613	def Dump(self):
614	# type: () -> Dict[str, value_t]
615	items = [value.Str(s) for s in self.argv] # type: List[value_t]
616	argv = value.List(items)
617	return {
618	'argv': argv,
619	'num_shifted': num.ToBig(self.num_shifted),
620	}
621
622	def GetArgNum(self, arg_num):
623	# type: (int) -> value_t
624
625	# $0 is handled elsewhere
626	assert 1 <= arg_num, arg_num
627
628	index = self.num_shifted + arg_num - 1
629	if index >= len(self.argv):
630	return value.Undef
631
632	return value.Str(self.argv[index])
633
634	def GetArgv(self):
635	# type: () -> List[str]
636	return self.argv[self.num_shifted:]
637
638	def GetNumArgs(self):
639	# type: () -> int
640	return len(self.argv) - self.num_shifted
641
642	def SetArgv(self, argv):
643	# type: (List[str]) -> None
644	self.argv = argv
645	self.num_shifted = 0
646
647
648	def _DumpVarFrame(frame):
649	# type: (Dict[str, Cell]) -> Dict[str, value_t]
650	"""Dump the stack frame as reasonably compact and readable JSON."""
651
652	vars_json = {} # type: Dict[str, value_t]
653	for name, cell in iteritems(frame):
654	cell_json = {} # type: Dict[str, value_t]
655
656	buf = mylib.BufWriter()
657	if cell.exported:
658	buf.write('x')
659	if cell.readonly:
660	buf.write('r')
661	flags = buf.getvalue()
662	if len(flags):
663	cell_json['flags'] = value.Str(flags)
664
665	# TODO:
666	# - Use packle for crash dumps! Then we can represent object cycles
667	# - Right now the JSON serializer will probably crash
668	# - although BashArray and BashAssoc may need 'type' tags
669	# - they don't round trip correctly
670	# - maybe add value.Tombstone here or something?
671	# - value.{Func,Eggex,...} may have value.Tombstone and
672	# vm.ValueIdString()?
673
674	with tagswitch(cell.val) as case:
675	if case(value_e.Undef):
676	cell_json['val'] = value.Null
677
678	elif case(value_e.Str, value_e.BashArray, value_e.BashAssoc,
679	value_e.SparseArray):
680	cell_json['val'] = cell.val
681
682	else:
683	# TODO: should we show the object ID here?
684	pass
685
686	vars_json[name] = value.Dict(cell_json)
687
688	return vars_json
689
690
691	def _LineNumber(tok):
692	# type: (Optional[Token]) -> str
693	""" For $BASH_LINENO """
694	if tok is None:
695	return '-1'
696	return str(tok.line.line_num)
697
698
699	def _AddCallToken(d, token):
700	# type: (Dict[str, value_t], Optional[Token]) -> None
701	if token is None:
702	return
703	d['call_source'] = value.Str(ui.GetLineSourceString(token.line))
704	d['call_line_num'] = num.ToBig(token.line.line_num)
705	d['call_line'] = value.Str(token.line.content)
706
707
708	class ctx_FuncCall(object):
709	"""For func calls."""
710
711	def __init__(self, mem, func):
712	# type: (Mem, value.Func) -> None
713
714	self.saved_globals = mem.var_stack[0]
715
716	assert func.module_frame is not None
717	mem.var_stack[0] = func.module_frame
718
719	frame = NewDict() # type: Dict[str, Cell]
720	mem.var_stack.append(frame)
721
722	mem.PushCall(func.name, func.parsed.name)
723
724	self.mem = mem
725
726	def __enter__(self):
727	# type: () -> None
728	pass
729
730	def __exit__(self, type, value, traceback):
731	# type: (Any, Any, Any) -> None
732	self.mem.PopCall()
733	self.mem.var_stack.pop()
734
735	self.mem.var_stack[0] = self.saved_globals
736
737
738	class ctx_ProcCall(object):
739	"""For proc calls, including shell functions."""
740
741	def __init__(self, mem, mutable_opts, proc, argv):
742	# type: (Mem, MutableOpts, value.Proc, List[str]) -> None
743
744	# TODO:
745	# should we separate procs and shell functions?
746	# - dynamic scope is one difference
747	# - '$@" shift etc. are another difference
748
749	self.saved_globals = mem.var_stack[0]
750
751	assert proc.module_frame is not None
752	mem.var_stack[0] = proc.module_frame
753
754	frame = NewDict() # type: Dict[str, Cell]
755
756	assert argv is not None
757	if proc.sh_compat:
758	# shell function
759	mem.argv_stack.append(_ArgFrame(argv))
760	else:
761	# procs
762	# - open: is equivalent to ...ARGV
763	# - closed: ARGV is empty list
764	frame['ARGV'] = _MakeArgvCell(argv)
765
766	mem.var_stack.append(frame)
767
768	mem.PushCall(proc.name, proc.name_tok)
769
770	# Dynamic scope is only for shell functions
771	mutable_opts.PushDynamicScope(proc.sh_compat)
772
773	# It may have been disabled with ctx_ErrExit for 'if echo $(false)', but
774	# 'if p' should be allowed.
775	self.mem = mem
776	self.mutable_opts = mutable_opts
777	self.sh_compat = proc.sh_compat
778
779	def __enter__(self):
780	# type: () -> None
781	pass
782
783	def __exit__(self, type, value, traceback):
784	# type: (Any, Any, Any) -> None
785	self.mutable_opts.PopDynamicScope()
786	self.mem.PopCall()
787	self.mem.var_stack.pop()
788
789	if self.sh_compat:
790	self.mem.argv_stack.pop()
791
792	self.mem.var_stack[0] = self.saved_globals
793
794
795	class ctx_Temp(object):
796	""" POSIX shell FOO=bar mycommand """
797
798	def __init__(self, mem):
799	# type: (Mem) -> None
800	self.mem = mem
801	mem.PushTemp()
802
803	def __enter__(self):
804	# type: () -> None
805	pass
806
807	def __exit__(self, type, value, traceback):
808	# type: (Any, Any, Any) -> None
809	self.mem.PopTemp()
810
811
812	class ctx_EnvObj(object):
813	"""YSH FOO=bar my-command"""
814
815	def __init__(self, mem, bindings):
816	# type: (Mem, Dict[str, value_t]) -> None
817	self.mem = mem
818	mem.PushEnvObj(bindings)
819
820	def __enter__(self):
821	# type: () -> None
822	pass
823
824	def __exit__(self, type, value, traceback):
825	# type: (Any, Any, Any) -> None
826	self.mem.PopEnvObj()
827
828
829	class ctx_Registers(object):
830	"""For $PS1, $PS4, $PROMPT_COMMAND, traps, and headless EVAL.
831
832	This is tightly coupled to state.Mem, so it's not in builtin/pure_ysh.
833	"""
834
835	def __init__(self, mem):
836	# type: (Mem) -> None
837
838	# Because some prompts rely on the status leaking. See issue #853.
839	# PS1 also does.
840	last = mem.last_status[-1]
841	mem.last_status.append(last)
842	mem.try_status.append(0)
843	mem.try_error.append(value.Dict({}))
844
845	# TODO: We should also copy these values! Turn the whole thing into a
846	# frame.
847	mem.pipe_status.append([])
848	mem.process_sub_status.append([])
849
850	mem.regex_match.append(regex_match.No)
851
852	self.mem = mem
853
854	def __enter__(self):
855	# type: () -> None
856	pass
857
858	def __exit__(self, type, value, traceback):
859	# type: (Any, Any, Any) -> None
860	self.mem.regex_match.pop()
861
862	self.mem.process_sub_status.pop()
863	self.mem.pipe_status.pop()
864
865	self.mem.try_error.pop()
866	self.mem.try_status.pop()
867	self.mem.last_status.pop()
868
869
870	class ctx_ThisDir(object):
871	"""For $_this_dir."""
872
873	def __init__(self, mem, filename):
874	# type: (Mem, Optional[str]) -> None
875	self.do_pop = False
876	if filename is not None: # script_name in main() may be -c, etc.
877	d = os_path.dirname(os_path.abspath(filename))
878	mem.this_dir.append(d)
879	self.do_pop = True
880
881	self.mem = mem
882
883	def __enter__(self):
884	# type: () -> None
885	pass
886
887	def __exit__(self, type, value, traceback):
888	# type: (Any, Any, Any) -> None
889	if self.do_pop:
890	self.mem.this_dir.pop()
891
892
893	def _MakeArgvCell(argv):
894	# type: (List[str]) -> Cell
895	items = [value.Str(a) for a in argv] # type: List[value_t]
896	return Cell(False, False, False, value.List(items))
897
898
899	class ctx_LoopFrame(object):
900
901	def __init__(self, mem, do_new_frame):
902	# type: (Mem, bool) -> None
903	self.mem = mem
904	self.do_new_frame = do_new_frame
905
906	if self.do_new_frame:
907	to_enclose = self.mem.var_stack[-1]
908	self.new_frame = NewDict() # type: Dict[str, Cell]
909	self.new_frame['__E__'] = Cell(False, False, False,
910	value.Frame(to_enclose))
911	mem.var_stack.append(self.new_frame)
912
913	def __enter__(self):
914	# type: () -> None
915	pass
916
917	def __exit__(self, type, value, traceback):
918	# type: (Any, Any, Any) -> None
919	if self.do_new_frame:
920	self.mem.var_stack.pop()
921
922
923	class ctx_EnclosedFrame(object):
924	"""
925	Usages:
926
927	- io->evalToDict(), which is a primitive used for Hay and the Dict proc
928	- lexical scope aka static scope for block args to user-defined procs
929	- Including the "closures in a loop" problem, which will be used for Hay
930
931	var mutated = 'm'
932	var shadowed = 's'
933
934	Dict (&d) {
935	shadowed = 42
936	mutated = 'new' # this is equivalent to var mutated
937
938	setvar mutated = 'new'
939	}
940	echo $shadowed # restored to 's'
941	echo $mutated # new
942
943	Or maybe we disallow the setvar lookup?
944	"""
945
946	def __init__(
947	self,
948	mem, # type: Mem
949	to_enclose, # type: Dict[str, Cell]
950	module_frame, # type: Dict[str, Cell]
951	out_dict, # type: Optional[Dict[str, value_t]]
952	):
953	# type: (...) -> None
954	self.mem = mem
955	self.to_enclose = to_enclose
956	self.module_frame = module_frame
957	self.out_dict = out_dict
958
959	if module_frame is not None:
960	self.saved_globals = self.mem.var_stack[0]
961	self.mem.var_stack[0] = module_frame
962
963	# __E__ gets a lookup rule
964	self.new_frame = NewDict() # type: Dict[str, Cell]
965	self.new_frame['__E__'] = Cell(False, False, False,
966	value.Frame(to_enclose))
967
968	mem.var_stack.append(self.new_frame)
969
970	def __enter__(self):
971	# type: () -> None
972	pass
973
974	def __exit__(self, type, value, traceback):
975	# type: (Any, Any, Any) -> None
976
977	if self.out_dict is not None:
978	for name, cell in iteritems(self.new_frame):
979	#log('name %r', name)
980	#log('cell %r', cell)
981
982	# User can hide variables with _ suffix
983	# e.g. for i_ in foo bar { echo $i_ }
984	if name.endswith('_'):
985	continue
986
987	self.out_dict[name] = cell.val
988
989	# Restore
990	self.mem.var_stack.pop()
991
992	if self.module_frame is not None:
993	self.mem.var_stack[0] = self.saved_globals
994
995
996	class ctx_ModuleEval(object):
997	"""Evaluate a module with a new global stack frame.
998
999	e.g. setglobal in the new module doesn't leak
1000
1001	Different from ctx_EnclosedFrame because the new code can't see variables in
1002	the old frame.
1003	"""
1004
1005	def __init__(self, mem, out_dict, out_errors):
1006	# type: (Mem, Dict[str, value_t], List[str]) -> None
1007	self.mem = mem
1008	self.out_dict = out_dict
1009	self.out_errors = out_errors
1010
1011	self.new_frame = NewDict() # type: Dict[str, Cell]
1012	self.saved_frame = mem.var_stack[0]
1013
1014	# Somewhat of a hack for tracing within a module.
1015	# Other solutions:
1016	# - PS4 can be __builtin__, but that would break shell compatibility
1017	# - We can have a separate YSH mechanism that uses a different settings
1018	# - We probably still want it to be scoped, like shvar PS4=z { ... }
1019	#
1020	# Note: there's a similar issue with HOSTNAME UID EUID etc. But those
1021	# could be io.hostname() io.getuid(), or lazy constants, etc.
1022
1023	ps4 = self.saved_frame.get('PS4')
1024	if ps4:
1025	self.new_frame['PS4'] = ps4
1026	# ENV is not in __builtins__ because it's mutable -- we want
1027	# 'setglobal' to work
1028	env = self.saved_frame.get('ENV')
1029	if env:
1030	self.new_frame['ENV'] = env
1031
1032	assert len(mem.var_stack) == 1
1033	mem.var_stack[0] = self.new_frame
1034
1035	# Whenever we're use-ing, the 'is-main' builtin will return 1 (false)
1036	self.to_restore = self.mem.is_main
1037	self.mem.is_main = False
1038
1039	# Equivalent of PushSource()
1040	# Would be nice to have a more precise location
1041	mem.debug_stack.append(debug_frame.Use(mem.token_for_line))
1042
1043	def __enter__(self):
1044	# type: () -> None
1045	pass
1046
1047	def __exit__(self, type, value_, traceback):
1048	# type: (Any, Any, Any) -> None
1049
1050	self.mem.debug_stack.pop()
1051
1052	self.mem.is_main = self.to_restore
1053
1054	assert len(self.mem.var_stack) == 1
1055	self.mem.var_stack[0] = self.saved_frame
1056
1057	# Now look in __export__ for the list of names to expose
1058
1059	cell = self.new_frame.get('__provide__')
1060	if cell is None:
1061	self.out_errors.append("Module is missing __provide__ List")
1062	return
1063
1064	provide_val = cell.val
1065	with tagswitch(provide_val) as case:
1066	if case(value_e.List):
1067	for val in cast(value.List, provide_val).items:
1068	if val.tag() == value_e.Str:
1069	name = cast(value.Str, val).s
1070
1071	cell = self.new_frame.get(name)
1072	if cell is None:
1073	self.out_errors.append(
1074	"Name %r was provided, but not defined" % name)
1075	continue
1076
1077	self.out_dict[name] = cell.val
1078	else:
1079	self.out_errors.append(
1080	"Expected Str in __provide__ List, got %s" %
1081	ui.ValType(val))
1082
1083	else:
1084	self.out_errors.append("__provide__ should be a List, got %s" %
1085	ui.ValType(provide_val))
1086
1087
1088	class ctx_Eval(object):
1089	"""Push temporary set of variables, $0, $1, $2, etc."""
1090
1091	def __init__(
1092	self,
1093	mem, # type: Mem
1094	dollar0, # type: Optional[str]
1095	pos_args, # type: Optional[List[str]]
1096	vars, # type: Optional[Dict[str, value_t]]
1097	):
1098	# type: (...) -> None
1099	self.mem = mem
1100	self.dollar0 = dollar0
1101	self.pos_args = pos_args
1102	self.vars = vars
1103
1104	# $0 needs to have lexical scoping. So we store it with other locals.
1105	# As "0" cannot be parsed as an lvalue, we can safely store dollar0 there.
1106	if dollar0 is not None:
1107	#assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef
1108	#self.dollar0_lval = LeftName("0", loc.Missing)
1109	#mem.SetLocalName(self.dollar0_lval, value.Str(dollar0))
1110
1111	self.restore_dollar0 = self.mem.dollar0
1112	self.mem.dollar0 = dollar0
1113
1114	if pos_args is not None:
1115	mem.argv_stack.append(_ArgFrame(pos_args))
1116
1117	if vars is not None:
1118	self.restore = [] # type: List[Tuple[LeftName, value_t]]
1119	self._Push(vars)
1120
1121	def __enter__(self):
1122	# type: () -> None
1123	pass
1124
1125	def __exit__(self, type, value_, traceback):
1126	# type: (Any, Any, Any) -> None
1127	if self.vars is not None:
1128	self._Pop()
1129
1130	if self.pos_args is not None:
1131	self.mem.argv_stack.pop()
1132
1133	if self.dollar0 is not None:
1134	#self.mem.SetLocalName(self.dollar0_lval, value.Undef)
1135	self.mem.dollar0 = self.restore_dollar0
1136
1137	# Note: _Push and _Pop are separate methods because the C++ translation
1138	# doesn't like when they are inline in __init__ and __exit__.
1139	def _Push(self, vars):
1140	# type: (Dict[str, value_t]) -> None
1141	for name in vars:
1142	lval = location.LName(name)
1143	# LocalOnly because we are only overwriting the current scope
1144	old_val = self.mem.GetValue(name, scope_e.LocalOnly)
1145	self.restore.append((lval, old_val))
1146	self.mem.SetNamed(lval, vars[name], scope_e.LocalOnly)
1147
1148	def _Pop(self):
1149	# type: () -> None
1150	for lval, old_val in self.restore:
1151	if old_val.tag() == value_e.Undef:
1152	self.mem.Unset(lval, scope_e.LocalOnly)
1153	else:
1154	self.mem.SetNamed(lval, old_val, scope_e.LocalOnly)
1155
1156
1157	def _FrameLookup(frame, name):
1158	# type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]]
1159	"""
1160	Look for a name in the frame, then recursively into the enclosing __E__
1161	frame, if it exists
1162	"""
1163	cell = frame.get(name)
1164	if cell:
1165	return cell, frame
1166
1167	rear_cell = frame.get('__E__') # ctx_EnclosedFrame() sets this
1168	if rear_cell:
1169	rear_val = rear_cell.val
1170	assert rear_val, rear_val
1171	if rear_val.tag() == value_e.Frame:
1172	to_enclose = cast(value.Frame, rear_val).frame
1173	return _FrameLookup(to_enclose, name) # recursive call
1174
1175	return None, None
1176
1177
1178	class Mem(object):
1179	"""For storing variables.
1180
1181	Callers:
1182	User code: assigning and evaluating variables, in command context or
1183	arithmetic context.
1184	Completion engine: for COMP_WORDS, etc.
1185	Builtins call it implicitly: read, cd for $PWD, $OLDPWD, etc.
1186
1187	Modules: cmd_eval, word_eval, expr_eval, completion
1188	"""
1189
1190	def __init__(
1191	self,
1192	dollar0, # type: str
1193	argv, # type: List[str]
1194	arena, # type: alloc.Arena
1195	debug_stack, # type: List[debug_frame_t]
1196	env_dict, # type: Dict[str, value_t]
1197	defaults=None, # type: Dict[str, value_t]
1198	):
1199	# type: (...) -> None
1200	"""
1201	Args:
1202	arena: currently unused
1203	"""
1204	# circular dep initialized out of line
1205	self.exec_opts = None # type: optview.Exec
1206	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
1207
1208	self.dollar0 = dollar0
1209	# If you only use YSH procs and funcs, this will remain at length 1.
1210	self.argv_stack = [_ArgFrame(argv)]
1211
1212	frame = NewDict() # type: Dict[str, Cell]
1213
1214	frame['ARGV'] = _MakeArgvCell(argv)
1215
1216	self.var_stack = [frame]
1217
1218	# The debug_stack isn't strictly necessary for execution. We use it
1219	# for crash dumps and for 3 parallel arrays: BASH_SOURCE, FUNCNAME, and
1220	# BASH_LINENO.
1221	self.debug_stack = debug_stack
1222
1223	self.env_dict = env_dict
1224	self.env_object = Obj(None, env_dict) # initial state
1225
1226	if defaults is None: # for unit tests only
1227	self.defaults = NewDict() # type: Dict[str, value_t]
1228	else:
1229	self.defaults = defaults
1230
1231	self.pwd = None # type: Optional[str]
1232	self.seconds_start = time_.time()
1233
1234	self.token_for_line = None # type: Optional[Token]
1235	self.loc_for_expr = loc.Missing # type: loc_t
1236
1237	self.last_arg = '' # $_ is initially empty, NOT unset
1238	self.line_num = value.Str('')
1239
1240	# Done ONCE on initialization
1241	self.root_pid = posix.getpid()
1242
1243	# TODO:
1244	# - These are REGISTERS mutated by user code.
1245	# - Call it self.reg_stack? with ctx_Registers
1246	# - push-registers builtin
1247	self.last_status = [0] # type: List[int] # a stack
1248	self.try_status = [0] # type: List[int] # a stack
1249	self.try_error = [value.Dict({})] # type: List[value.Dict] # a stack
1250	self.pipe_status = [[]] # type: List[List[int]] # stack
1251	self.process_sub_status = [[]] # type: List[List[int]] # stack
1252
1253	# A stack but NOT a register?
1254	self.this_dir = [] # type: List[str]
1255	self.regex_match = [regex_match.No] # type: List[regex_match_t]
1256
1257	self.last_bg_pid = -1 # Uninitialized value mutable public variable
1258
1259	self.running_debug_trap = False # set by ctx_DebugTrap()
1260	self.running_err_trap = False # set by ctx_ErrTrap
1261	self.is_main = True # we start out in main
1262
1263	# For the ctx builtin
1264	self.ctx_stack = [] # type: List[Dict[str, value_t]]
1265
1266	self.builtins = NewDict() # type: Dict[str, value_t]
1267
1268	# Note: Python 2 and 3 have __builtins__
1269	# This is just for inspection
1270	builtins_module = Obj(None, self.builtins)
1271
1272	# Code in any module can see __builtins__
1273	self.builtins['__builtins__'] = builtins_module
1274
1275	self.did_ysh_env = False # only initialize ENV once per process
1276
1277	from core import sh_init
1278	self.env_config = sh_init.EnvConfig(self, defaults)
1279
1280	def __repr__(self):
1281	# type: () -> str
1282	parts = [] # type: List[str]
1283	parts.append('<Mem')
1284	for i, frame in enumerate(self.var_stack):
1285	parts.append(' -- %d --' % i)
1286	for n, v in frame.iteritems():
1287	parts.append(' %s %s' % (n, v))
1288	parts.append('>')
1289	return '\n'.join(parts) + '\n'
1290
1291	def AddBuiltin(self, name, val):
1292	# type: (str, value_t) -> None
1293	self.builtins[name] = val
1294
1295	def SetPwd(self, pwd):
1296	# type: (str) -> None
1297	"""Used by builtins."""
1298	self.pwd = pwd
1299
1300	def ParsingChangesAllowed(self):
1301	# type: () -> bool
1302	"""For checking that syntax options are only used at the top level."""
1303
1304	# DISALLOW proc calls : they push argv_stack, var_stack, debug_stack
1305	# ALLOW source foo.sh arg1: pushes argv_stack, debug_stack
1306	# ALLOW FOO=bar : pushes var_stack
1307	return len(self.var_stack) == 1 or len(self.argv_stack) == 1
1308
1309	def Dump(self):
1310	# type: () -> Tuple[List[value_t], List[value_t], List[value_t]]
1311	"""Copy state before unwinding the stack."""
1312	var_stack = [
1313	value.Dict(_DumpVarFrame(frame)) for frame in self.var_stack
1314	] # type: List[value_t]
1315	argv_stack = [value.Dict(frame.Dump())
1316	for frame in self.argv_stack] # type: List[value_t]
1317
1318	debug_stack = [] # type: List[value_t]
1319
1320	# Reuse these immutable objects
1321	t_call = value.Str('Call')
1322	t_source = value.Str('Source')
1323
1324	for frame in reversed(self.debug_stack):
1325	UP_frame = frame
1326	d = None # type: Optional[Dict[str, value_t]]
1327	with tagswitch(frame) as case:
1328	if case(debug_frame_e.Call):
1329	frame = cast(debug_frame.Call, UP_frame)
1330	d = {
1331	'type': t_call,
1332	'func_name': value.Str(frame.func_name)
1333	}
1334
1335	_AddCallToken(d, frame.call_tok)
1336	# TODO: Add def_tok
1337
1338	elif case(debug_frame_e.Source):
1339	frame = cast(debug_frame.Source, UP_frame)
1340	d = {
1341	'type': t_source,
1342	'source_name': value.Str(frame.source_name)
1343	}
1344	_AddCallToken(d, frame.call_tok)
1345
1346	# Note: Skip debug_frame.MainFile
1347	if d is not None:
1348	debug_stack.append(value.Dict(d))
1349
1350	return var_stack, argv_stack, debug_stack
1351
1352	def SetLastArgument(self, s):
1353	# type: (str) -> None
1354	"""For $_"""
1355	self.last_arg = s
1356
1357	def SetTokenForLine(self, tok):
1358	# type: (Token) -> None
1359	"""Set a token to compute $LINENO
1360
1361	This means it should be set on SimpleCommand, ShAssignment, ((, [[,
1362	case, etc. -- anything that evaluates a word. Example: there was a bug
1363	with 'case $LINENO'
1364
1365	This token also used as a "least-specific" / fallback location for
1366	errors in ExecuteAndCatch().
1367
1368	Although most of that should be taken over by 'with ui.ctx_Location()`,
1369	for the errfmt.
1370	"""
1371	if self.running_debug_trap or self.running_err_trap:
1372	return
1373
1374	#if tok.span_id == runtime.NO_SPID:
1375	# NOTE: This happened in the osh-runtime benchmark for yash.
1376	#log('Warning: span_id undefined in SetTokenForLine')
1377
1378	#import traceback
1379	#traceback.print_stack()
1380	#return
1381
1382	# Reset the expression fallback location on every line. It REFINES the
1383	# line-based fallback location.
1384	self.loc_for_expr = loc.Missing
1385
1386	self.token_for_line = tok
1387
1388	def SetLocationForExpr(self, blame_loc):
1389	# type: (loc_t) -> None
1390	"""
1391	A more specific fallback location, like the $[ in
1392
1393	echo $[len(42)]
1394	"""
1395	self.loc_for_expr = blame_loc
1396
1397	def GetFallbackLocation(self):
1398	# type: () -> loc_t
1399
1400	if self.loc_for_expr != loc.Missing: # more specific
1401	return self.loc_for_expr
1402
1403	if self.token_for_line: # less specific
1404	return self.token_for_line
1405
1406	return loc.Missing
1407
1408	#
1409	# Status Variable Stack (for isolating $PS1 and $PS4)
1410	#
1411
1412	def LastStatus(self):
1413	# type: () -> int
1414	return self.last_status[-1]
1415
1416	def TryStatus(self):
1417	# type: () -> int
1418	return self.try_status[-1]
1419
1420	def TryError(self):
1421	# type: () -> value.Dict
1422	return self.try_error[-1]
1423
1424	def PipeStatus(self):
1425	# type: () -> List[int]
1426	return self.pipe_status[-1]
1427
1428	def SetLastStatus(self, x):
1429	# type: (int) -> None
1430	self.last_status[-1] = x
1431
1432	def SetTryStatus(self, x):
1433	# type: (int) -> None
1434	self.try_status[-1] = x
1435
1436	def SetTryError(self, x):
1437	# type: (value.Dict) -> None
1438	self.try_error[-1] = x
1439
1440	def SetPipeStatus(self, x):
1441	# type: (List[int]) -> None
1442	self.pipe_status[-1] = x
1443
1444	def SetSimplePipeStatus(self, status):
1445	# type: (int) -> None
1446
1447	# Optimization to avoid allocations
1448	top = self.pipe_status[-1]
1449	if len(top) == 1:
1450	top[0] = status
1451	else:
1452	self.pipe_status[-1] = [status]
1453
1454	def SetProcessSubStatus(self, x):
1455	# type: (List[int]) -> None
1456	self.process_sub_status[-1] = x
1457
1458	#
1459	# Call Stack
1460	#
1461
1462	def PushCall(self, func_name, def_tok):
1463	# type: (str, Token) -> None
1464	"""Push argv, var, and debug stack frames.
1465
1466	Currently used for proc and func calls. TODO: New func evaluator may
1467	not use it.
1468
1469	Args:
1470	def_tok: Token where proc or func was defined, used to compute
1471	BASH_SOURCE.
1472	"""
1473	self.debug_stack.append(
1474	debug_frame.Call(self.token_for_line, def_tok, func_name))
1475
1476	def PopCall(self):
1477	# type: () -> None
1478	"""
1479	Args:
1480	should_pop_argv_stack: Pass False if PushCall was given None for argv
1481	True for proc, False for func
1482	"""
1483	self.debug_stack.pop()
1484
1485	def ShouldRunDebugTrap(self):
1486	# type: () -> bool
1487
1488	# TODO: RunLastPart of pipeline can disable this
1489
1490	# Don't recursively run DEBUG trap
1491	if self.running_debug_trap:
1492	return False
1493
1494	# Don't run it inside functions
1495	if len(self.var_stack) > 1:
1496	return False
1497
1498	return True
1499
1500	def IsGlobalScope(self):
1501	# type: () -> bool
1502	"""
1503	local -g uses this, probably because bash does the wrong thing and
1504	prints LOCALS, not globals.
1505	"""
1506	return len(self.var_stack) == 1
1507
1508	def InsideFunction(self):
1509	# type: () -> bool
1510	"""For the ERR trap, and use builtin"""
1511
1512	# TODO: Should this be unified with ParsingChangesAllowed()? Slightly
1513	# different logic.
1514
1515	# Don't run it inside functions
1516	return len(self.var_stack) > 1
1517
1518	def GlobalFrame(self):
1519	# type: () -> Dict[str, Cell]
1520	"""For defining the global scope of modules.
1521
1522	It's affected by ctx_ModuleEval()
1523	"""
1524	return self.var_stack[0]
1525
1526	def CurrentFrame(self):
1527	# type: () -> Dict[str, Cell]
1528	"""For attaching a stack frame to a value.Block"""
1529	return self.var_stack[-1]
1530
1531	def PushSource(self, source_name, argv):
1532	# type: (str, List[str]) -> None
1533	""" For 'source foo.sh 1 2 3' """
1534	if len(argv):
1535	self.argv_stack.append(_ArgFrame(argv))
1536
1537	# self.token_for_line can be None?
1538	self.debug_stack.append(
1539	debug_frame.Source(self.token_for_line, source_name))
1540
1541	def PopSource(self, argv):
1542	# type: (List[str]) -> None
1543	self.debug_stack.pop()
1544
1545	if len(argv):
1546	self.argv_stack.pop()
1547
1548	def PushTemp(self):
1549	# type: () -> None
1550	"""For the temporary scope in 'FOO=bar BAR=baz echo'.
1551
1552	Also for PS4 evaluation with more variables.
1553	"""
1554	# We don't want the 'read' builtin to write to this frame!
1555	frame = NewDict() # type: Dict[str, Cell]
1556	self.var_stack.append(frame)
1557
1558	def PopTemp(self):
1559	# type: () -> None
1560	self.var_stack.pop()
1561
1562	def _BindEnvObj(self):
1563	# type: () -> None
1564	self.SetNamed(location.LName('ENV'), self.env_object,
1565	scope_e.GlobalOnly)
1566
1567	def MaybeInitEnvDict(self, environ):
1568	# type: (Dict[str, str]) -> None
1569	if self.did_ysh_env:
1570	return
1571
1572	for name, s in iteritems(environ):
1573	self.env_dict[name] = value.Str(s)
1574
1575	self._BindEnvObj()
1576	self.did_ysh_env = True
1577
1578	def PushEnvObj(self, bindings):
1579	# type: (Dict[str, value_t]) -> None
1580	"""Push "bindings" as the MOST visible part of the ENV Obj
1581
1582	i.e. first() / propView()
1583	"""
1584	self.env_object = Obj(self.env_object, bindings)
1585	self._BindEnvObj()
1586
1587	def PopEnvObj(self):
1588	# type: () -> None
1589	"""Pop a Dict of bindings."""
1590	self.env_object = self.env_object.prototype
1591	if self.env_object is None:
1592	# Note: there isn't a way to hit this now, but let's be defensive.
1593	# See test case in spec/ysh-env.test.sh.
1594	e_die('PopEnvObj: env.prototype is null', loc.Missing)
1595
1596	self._BindEnvObj()
1597
1598	#
1599	# Argv
1600	#
1601
1602	def Shift(self, n):
1603	# type: (int) -> int
1604	frame = self.argv_stack[-1]
1605	num_args = len(frame.argv)
1606
1607	if (frame.num_shifted + n) <= num_args:
1608	frame.num_shifted += n
1609	return 0 # success
1610	else:
1611	return 1 # silent error
1612
1613	def GetArg0(self):
1614	# type: () -> value.Str
1615	"""Like GetArgNum(0) but with a more specific type."""
1616	return value.Str(self.dollar0)
1617
1618	def GetArgNum(self, arg_num):
1619	# type: (int) -> value_t
1620	if arg_num == 0:
1621	# Disabled
1622	if 0:
1623	# Problem: Doesn't obey enclosing frame?
1624	# Yeah it needs FrameLookup
1625	cell, _ = _FrameLookup(self.var_stack[-1], '0')
1626	if cell is not None:
1627	val = cell.val
1628	if val.tag() != value_e.Undef:
1629	return val
1630
1631	return value.Str(self.dollar0)
1632
1633	return self.argv_stack[-1].GetArgNum(arg_num)
1634
1635	def GetArgv(self):
1636	# type: () -> List[str]
1637	"""For $* and $@."""
1638	return self.argv_stack[-1].GetArgv()
1639
1640	def SetArgv(self, argv):
1641	# type: (List[str]) -> None
1642	"""For set -- 1 2 3."""
1643	# from set -- 1 2 3
1644	self.argv_stack[-1].SetArgv(argv)
1645
1646	#
1647	# Special Vars
1648	#
1649
1650	def GetSpecialVar(self, op_id):
1651	# type: (int) -> value_t
1652	if op_id == Id.VSub_Bang: # $!
1653	n = self.last_bg_pid
1654	if n == -1:
1655	return value.Undef # could be an error
1656
1657	elif op_id == Id.VSub_QMark: # $?
1658	# External commands need WIFEXITED test. What about subshells?
1659	n = self.last_status[-1]
1660
1661	elif op_id == Id.VSub_Pound: # $#
1662	n = self.argv_stack[-1].GetNumArgs()
1663
1664	elif op_id == Id.VSub_Dollar: # $$
1665	n = self.root_pid
1666
1667	else:
1668	raise NotImplementedError(op_id)
1669
1670	return value.Str(str(n))
1671
1672	#
1673	# Named Vars
1674	#
1675
1676	def _ResolveNameOnly(self, name, which_scopes):
1677	# type: (str, scope_t) -> Tuple[Optional[Cell], Dict[str, Cell]]
1678	"""Helper for getting and setting variable.
1679
1680	Returns:
1681	cell: The cell corresponding to looking up 'name' with the given mode, or
1682	None if it's not found.
1683	var_frame: The frame it should be set to or deleted from.
1684	"""
1685	if which_scopes == scope_e.Dynamic:
1686	for i in xrange(len(self.var_stack) - 1, -1, -1):
1687	var_frame = self.var_stack[i]
1688	cell, result_frame = _FrameLookup(var_frame, name)
1689	if cell:
1690	return cell, result_frame
1691	return None, self.var_stack[0] # set in global var_frame
1692
1693	if which_scopes == scope_e.LocalOnly:
1694	var_frame = self.var_stack[-1]
1695	cell, result_frame = _FrameLookup(var_frame, name)
1696	if cell:
1697	return cell, result_frame
1698	return None, var_frame
1699
1700	if which_scopes == scope_e.GlobalOnly:
1701	var_frame = self.var_stack[0]
1702	cell, result_frame = _FrameLookup(var_frame, name)
1703	if cell:
1704	return cell, result_frame
1705
1706	return None, var_frame
1707
1708	if which_scopes == scope_e.LocalOrGlobal:
1709	# Local
1710	var_frame = self.var_stack[-1]
1711	cell, result_frame = _FrameLookup(var_frame, name)
1712	if cell:
1713	return cell, result_frame
1714
1715	# Global
1716	var_frame = self.var_stack[0]
1717	cell, result_frame = _FrameLookup(var_frame, name)
1718	if cell:
1719	return cell, result_frame
1720
1721	return None, var_frame
1722
1723	raise AssertionError()
1724
1725	def _ResolveNameOrRef(
1726	self,
1727	name, # type: str
1728	which_scopes, # type: scope_t
1729	ref_trail=None, # type: Optional[List[str]]
1730	):
1731	# type: (...) -> Tuple[Optional[Cell], Dict[str, Cell], str]
1732	"""Look up a cell and namespace, but respect the nameref flag.
1733
1734	Resolving namerefs does RECURSIVE calls.
1735	"""
1736	cell, var_frame = self._ResolveNameOnly(name, which_scopes)
1737
1738	if cell is None or not cell.nameref:
1739	return cell, var_frame, name # not a nameref
1740
1741	val = cell.val
1742	UP_val = val
1743	with tagswitch(val) as case:
1744	if case(value_e.Undef):
1745	# This is 'local -n undef_ref', which is kind of useless, because the
1746	# more common idiom is 'local -n ref=$1'. Note that you can mutate
1747	# references themselves with local -n ref=new.
1748	if self.exec_opts.strict_nameref():
1749	e_die('nameref %r is undefined' % name)
1750	else:
1751	return cell, var_frame, name # fallback
1752
1753	elif case(value_e.Str):
1754	val = cast(value.Str, UP_val)
1755	new_name = val.s
1756
1757	else:
1758	# SetValue() protects the invariant that nameref is Undef or Str
1759	raise AssertionError(val.tag())
1760
1761	# TODO: Respect eval_unsafe_arith here (issue 881). See how it's done in
1762	# 'printf -v' with MakeArithParser
1763	if not match.IsValidVarName(new_name):
1764	# e.g. '#' or '1' or ''
1765	if self.exec_opts.strict_nameref():
1766	e_die('nameref %r contains invalid variable name %r' %
1767	(name, new_name))
1768	else:
1769	# Bash has this odd behavior of clearing the nameref bit when
1770	# ref=#invalid#. strict_nameref avoids it.
1771	cell.nameref = False
1772	return cell, var_frame, name # fallback
1773
1774	# Check for circular namerefs.
1775	if ref_trail is None:
1776	ref_trail = [name]
1777	else:
1778	if new_name in ref_trail:
1779	e_die('Circular nameref %s' % ' -> '.join(ref_trail))
1780	ref_trail.append(new_name)
1781
1782	# 'declare -n' uses dynamic scope.
1783	cell, var_frame, cell_name = self._ResolveNameOrRef(
1784	new_name, scope_e.Dynamic, ref_trail=ref_trail)
1785	return cell, var_frame, cell_name
1786
1787	def IsBashAssoc(self, name):
1788	# type: (str) -> bool
1789	"""Returns whether a name resolve to a cell with an associative array.
1790
1791	We need to know this to evaluate the index expression properly
1792	-- should it be coerced to an integer or not?
1793	"""
1794	cell, _, _ = self._ResolveNameOrRef(name, self.ScopesForReading())
1795	# foo=([key]=value)
1796	return cell is not None and cell.val.tag() == value_e.BashAssoc
1797
1798	def SetPlace(self, place, val, blame_loc):
1799	# type: (value.Place, value_t, loc_t) -> None
1800
1801	yval = place.lval
1802	UP_yval = yval
1803	with tagswitch(yval) as case:
1804	if case(y_lvalue_e.Local):
1805	yval = cast(LeftName, UP_yval)
1806
1807	if 0:
1808	# Check that the frame is still alive
1809	# Note: Disabled because it doesn't work with modules. the
1810	# Place captures a frame in def-test.ysh, which we want to
1811	# mutate while Dict is executing in the module_frame for
1812	# def.ysh. See ctx_ModuleEval
1813	found = False
1814	for i in xrange(len(self.var_stack) - 1, -1, -1):
1815	frame = self.var_stack[i]
1816	if frame is place.frame:
1817	found = True
1818	#log('FOUND %s', found)
1819	break
1820	if not found:
1821	e_die(
1822	"Can't assign to place that's no longer on the call stack.",
1823	blame_loc)
1824
1825	frame = place.frame
1826	cell = frame.get(yval.name)
1827	if cell is None:
1828	cell = Cell(False, False, False, val)
1829	frame[yval.name] = cell
1830	else:
1831	cell.val = val
1832
1833	elif case(y_lvalue_e.Container):
1834	e_die('Container place not implemented', blame_loc)
1835
1836	else:
1837	raise AssertionError()
1838
1839	def SetLocalName(self, lval, val):
1840	# type: (LeftName, value_t) -> None
1841	"""
1842	Set a name in the local scope - used for func/proc param binding, etc.
1843	"""
1844
1845	# Equivalent to
1846	# self._ResolveNameOnly(lval.name, scope_e.LocalOnly)
1847	var_frame = self.var_stack[-1]
1848	cell = var_frame.get(lval.name)
1849
1850	if cell:
1851	if cell.readonly:
1852	e_die("Can't assign to readonly value %r" % lval.name,
1853	lval.blame_loc)
1854	cell.val = val # Mutate value_t
1855	else:
1856	cell = Cell(False, False, False, val)
1857	var_frame[lval.name] = cell
1858
1859	def SetNamed(self, lval, val, which_scopes, flags=0):
1860	# type: (LeftName, value_t, scope_t, int) -> None
1861	if flags & SetNameref or flags & ClearNameref:
1862	# declare -n ref=x # refers to the ref itself
1863	cell, var_frame = self._ResolveNameOnly(lval.name, which_scopes)
1864	cell_name = lval.name
1865	else:
1866	# ref=x # mutates THROUGH the reference
1867
1868	# Note on how to implement declare -n ref='a[42]'
1869	# 1. Call _ResolveNameOnly()
1870	# 2. If cell.nameref, call self.unsafe_arith.ParseVarRef() ->
1871	# BracedVarSub
1872	# 3. Turn BracedVarSub into an sh_lvalue, and call
1873	# self.unsafe_arith.SetValue() wrapper with ref_trail
1874	cell, var_frame, cell_name = self._ResolveNameOrRef(
1875	lval.name, which_scopes)
1876
1877	if cell:
1878	# Clear before checking readonly bit.
1879	# NOTE: Could be cell.flags &= flag_clear_mask
1880	if flags & ClearExport:
1881	cell.exported = False
1882	if flags & ClearReadOnly:
1883	cell.readonly = False
1884	if flags & ClearNameref:
1885	cell.nameref = False
1886
1887	if val is not None: # e.g. declare -rx existing
1888	# Note: this DYNAMIC check means we can't have 'const' in a loop.
1889	# But that's true for 'readonly' too, and hoisting it makes more
1890	# sense anyway.
1891	if cell.readonly:
1892	e_die("Can't assign to readonly value %r" % lval.name,
1893	lval.blame_loc)
1894	cell.val = val # CHANGE VAL
1895
1896	# NOTE: Could be cell.flags \|= flag_set_mask
1897	if flags & SetExport:
1898	cell.exported = True
1899	if flags & SetReadOnly:
1900	cell.readonly = True
1901	if flags & SetNameref:
1902	cell.nameref = True
1903
1904	else:
1905	if val is None: # declare -rx nonexistent
1906	# set -o nounset; local foo; echo $foo # It's still undefined!
1907	val = value.Undef # export foo, readonly foo
1908
1909	cell = Cell(bool(flags & SetExport), bool(flags & SetReadOnly),
1910	bool(flags & SetNameref), val)
1911	var_frame[cell_name] = cell
1912
1913	# Maintain invariant that only strings and undefined cells can be
1914	# exported.
1915	assert cell.val is not None, cell
1916
1917	if cell.val.tag() not in (value_e.Undef, value_e.Str):
1918	if cell.exported:
1919	if self.exec_opts.strict_array():
1920	e_die("Only strings can be exported (strict_array)",
1921	lval.blame_loc)
1922	if cell.nameref:
1923	e_die("nameref must be a string", lval.blame_loc)
1924
1925	def SetValue(self, lval, val, which_scopes, flags=0):
1926	# type: (sh_lvalue_t, value_t, scope_t, int) -> None
1927	"""
1928	Args:
1929	lval: sh_lvalue
1930	val: value, or None if only changing flags
1931	which_scopes:
1932	Local \| Global \| Dynamic - for builtins, PWD, etc.
1933	flags: packed pair (keyword_id, bit mask of set/clear flags)
1934
1935	Note: in bash, PWD=/ changes the directory. But not in dash.
1936	"""
1937	# STRICTNESS / SANENESS:
1938	#
1939	# 1) Don't create arrays automatically, e.g. a[1000]=x
1940	# 2) Never change types? yeah I think that's a good idea, at least for YSH
1941	# (not sh, for compatibility). set -o strict_types or something. That
1942	# means arrays have to be initialized with let arr = [], which is fine.
1943	# This helps with stuff like IFS. It starts off as a string, and assigning
1944	# it to a list is an error. I guess you will have to turn this no for
1945	# bash?
1946	#
1947	# TODO:
1948	# - COMPUTED vars can't be set
1949	# - What about PWD / OLDPWD / UID / EUID ? You can simply make them
1950	# readonly.
1951	# - Maybe PARSE $PS1 and $PS4 when they're set, to avoid the error on use?
1952	# - Other validity: $HOME could be checked for existence
1953
1954	UP_lval = lval
1955	with tagswitch(lval) as case:
1956	if case(sh_lvalue_e.Var):
1957	lval = cast(LeftName, UP_lval)
1958
1959	self.SetNamed(lval, val, which_scopes, flags=flags)
1960
1961	elif case(sh_lvalue_e.Indexed):
1962	lval = cast(sh_lvalue.Indexed, UP_lval)
1963
1964	# There is no syntax 'declare a[x]'
1965	assert val is not None, val
1966
1967	# TODO: relax this for YSH
1968	assert val.tag() == value_e.Str, val
1969	rval = cast(value.Str, val)
1970
1971	# Note: location could be a[x]=1 or (( a[ x ] = 1 ))
1972	left_loc = lval.blame_loc
1973
1974	# bash/mksh have annoying behavior of letting you do LHS assignment to
1975	# Undef, which then turns into an INDEXED array. (Undef means that set
1976	# -o nounset fails.)
1977	cell, var_frame, _ = self._ResolveNameOrRef(
1978	lval.name, which_scopes)
1979	if not cell:
1980	self._BindNewArrayWithEntry(var_frame, lval, rval, flags)
1981	return
1982
1983	if cell.readonly:
1984	e_die("Can't assign to readonly array", left_loc)
1985
1986	UP_cell_val = cell.val
1987	# undef[0]=y is allowed
1988	with tagswitch(UP_cell_val) as case2:
1989	if case2(value_e.Undef):
1990	self._BindNewArrayWithEntry(var_frame, lval, rval,
1991	flags)
1992	return
1993
1994	elif case2(value_e.Str):
1995	# s=x
1996	# s[1]=y # invalid
1997	e_die("Can't assign to items in a string", left_loc)
1998
1999	elif case2(value_e.BashArray):
2000	cell_val = cast(value.BashArray, UP_cell_val)
2001	error_code = bash_impl.BashArray_SetElement(
2002	cell_val, lval.index, rval.s)
2003	if error_code == error_code_e.IndexOutOfRange:
2004	n = bash_impl.BashArray_Length(cell_val)
2005	e_die(
2006	"Index %d is out of bounds for array of length %d"
2007	% (lval.index, n), left_loc)
2008	return
2009
2010	elif case2(value_e.SparseArray):
2011	lhs_sp = cast(value.SparseArray, UP_cell_val)
2012	error_code = bash_impl.SparseArray_SetElement(
2013	lhs_sp, mops.IntWiden(lval.index), rval.s)
2014	if error_code == error_code_e.IndexOutOfRange:
2015	n_big = bash_impl.SparseArray_Length(lhs_sp)
2016	e_die(
2017	"Index %d is out of bounds for array of length %s"
2018	% (lval.index, mops.ToStr(n_big)), left_loc)
2019	return
2020
2021	# This could be an object, eggex object, etc. It won't be
2022	# BashAssoc shouldn because we query IsBashAssoc before evaluating
2023	# sh_lhs. Could conslidate with s[i] case above
2024	e_die(
2025	"Value of type %s can't be indexed" % ui.ValType(cell.val),
2026	left_loc)
2027
2028	elif case(sh_lvalue_e.Keyed):
2029	lval = cast(sh_lvalue.Keyed, UP_lval)
2030
2031	# There is no syntax 'declare A["x"]'
2032	assert val is not None, val
2033	assert val.tag() == value_e.Str, val
2034	rval = cast(value.Str, val)
2035
2036	left_loc = lval.blame_loc
2037
2038	cell, var_frame, _ = self._ResolveNameOrRef(
2039	lval.name, which_scopes)
2040	if cell.readonly:
2041	e_die("Can't assign to readonly associative array",
2042	left_loc)
2043
2044	# We already looked it up before making the sh_lvalue
2045	assert cell.val.tag() == value_e.BashAssoc, cell
2046	cell_val2 = cast(value.BashAssoc, cell.val)
2047	bash_impl.BashAssoc_SetElement(cell_val2, lval.key, rval.s)
2048
2049	else:
2050	raise AssertionError(lval.tag())
2051
2052	def _BindNewArrayWithEntry(self, var_frame, lval, val, flags):
2053	# type: (Dict[str, Cell], sh_lvalue.Indexed, value.Str, int) -> None
2054	"""Fill 'var_frame' with a new indexed array entry."""
2055	no_str = None # type: Optional[str]
2056	items = [no_str] * lval.index
2057	items.append(val.s)
2058	new_value = value.BashArray(items)
2059
2060	# arrays can't be exported; can't have BashAssoc flag
2061	readonly = bool(flags & SetReadOnly)
2062	var_frame[lval.name] = Cell(False, readonly, False, new_value)
2063
2064	def InternalSetGlobal(self, name, new_val):
2065	# type: (str, value_t) -> None
2066	"""For setting read-only globals internally.
2067
2068	Args:
2069	name: string (not Lhs)
2070	new_val: value
2071
2072	The variable must already exist.
2073
2074	Use case: SHELLOPTS.
2075	"""
2076	cell = self.var_stack[0][name]
2077	cell.val = new_val
2078
2079	def GetValue(self, name, which_scopes=scope_e.Shopt):
2080	# type: (str, scope_t) -> value_t
2081	"""Used by the WordEvaluator, ArithEvaluator, ExprEvaluator, etc."""
2082	assert isinstance(name, str), name
2083
2084	if which_scopes == scope_e.Shopt:
2085	which_scopes = self.ScopesForReading()
2086
2087	with str_switch(name) as case:
2088	# "Registers"
2089	if case('_status'): # deprecated in favor of _error.code
2090	return num.ToBig(self.TryStatus())
2091
2092	elif case('_error'):
2093	return self.TryError()
2094
2095	elif case('_this_dir'):
2096	if len(self.this_dir) == 0:
2097	# e.g. osh -c '' doesn't have it set
2098	# Should we give a custom error here?
2099	# If you're at the interactive shell, 'source mymodule.ysh' will still
2100	# work because 'source' sets it.
2101	return value.Undef
2102	else:
2103	return value.Str(self.this_dir[-1]) # top of stack
2104
2105	elif case('PIPESTATUS'):
2106	strs2 = [str(i)
2107	for i in self.pipe_status[-1]] # type: List[str]
2108	return value.BashArray(strs2)
2109
2110	elif case('_pipeline_status'):
2111	items = [num.ToBig(i)
2112	for i in self.pipe_status[-1]] # type: List[value_t]
2113	return value.List(items)
2114
2115	elif case('_process_sub_status'): # YSH naming convention
2116	items = [num.ToBig(i) for i in self.process_sub_status[-1]]
2117	return value.List(items)
2118
2119	elif case('BASH_REMATCH'):
2120	top_match = self.regex_match[-1]
2121	with tagswitch(top_match) as case2:
2122	if case2(regex_match_e.No):
2123	groups = [] # type: List[str]
2124	elif case2(regex_match_e.Yes):
2125	m = cast(RegexMatch, top_match)
2126	groups = util.RegexGroupStrings(m.s, m.indices)
2127	return value.BashArray(groups)
2128
2129	# Do lookup of system globals before looking at user variables. Note: we
2130	# could optimize this at compile-time like $?. That would break
2131	# ${!varref}, but it's already broken for $?.
2132
2133	elif case('FUNCNAME'):
2134	# bash wants it in reverse order. This is a little inefficient but we're
2135	# not depending on deque().
2136	strs = [] # type: List[str]
2137	for frame in reversed(self.debug_stack):
2138	UP_frame = frame
2139	with tagswitch(frame) as case2:
2140	if case2(debug_frame_e.Call):
2141	frame = cast(debug_frame.Call, UP_frame)
2142	strs.append(frame.func_name)
2143
2144	elif case2(debug_frame_e.Source):
2145	# bash doesn't tell you the filename sourced
2146	strs.append('source')
2147
2148	elif case2(debug_frame_e.MainFile):
2149	strs.append('main') # also bash behavior
2150
2151	else: # ignore
2152	pass
2153
2154	return value.BashArray(strs) # TODO: Reuse this object too?
2155
2156	# $BASH_SOURCE and $BASH_LINENO have OFF BY ONE design bugs:
2157	#
2158	# ${BASH_LINENO[$i]} is the line number in the source file
2159	# (${BASH_SOURCE[$i+1]}) where ${FUNCNAME[$i]} was called (or
2160	# ${BASH_LINENO[$i-1]} if referenced within another shell function).
2161	#
2162	# https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html
2163
2164	elif case('BASH_SOURCE'):
2165	strs = []
2166	for frame in reversed(self.debug_stack):
2167	UP_frame = frame
2168	with tagswitch(frame) as case2:
2169	if case2(debug_frame_e.Call):
2170	frame = cast(debug_frame.Call, UP_frame)
2171
2172	# Weird bash behavior
2173	assert frame.def_tok.line is not None
2174	source_str = ui.GetLineSourceString(
2175	frame.def_tok.line)
2176	strs.append(source_str)
2177
2178	elif case2(debug_frame_e.Source):
2179	frame = cast(debug_frame.Source, UP_frame)
2180	# Is this right?
2181	strs.append(frame.source_name)
2182
2183	elif case2(debug_frame_e.MainFile):
2184	frame = cast(debug_frame.MainFile, UP_frame)
2185	strs.append(frame.main_filename)
2186
2187	else: # ignore
2188	pass
2189
2190	return value.BashArray(strs) # TODO: Reuse this object too?
2191
2192	elif case('BASH_LINENO'):
2193	strs = []
2194	for frame in reversed(self.debug_stack):
2195	UP_frame = frame
2196	with tagswitch(frame) as case2:
2197	if case2(debug_frame_e.Call):
2198	frame = cast(debug_frame.Call, UP_frame)
2199	strs.append(_LineNumber(frame.call_tok))
2200
2201	elif case2(debug_frame_e.Source):
2202	frame = cast(debug_frame.Source, UP_frame)
2203	strs.append(_LineNumber(frame.call_tok))
2204
2205	elif case2(debug_frame_e.MainFile):
2206	# Bash does this to line up with 'main'
2207	strs.append('0')
2208
2209	else: # ignore
2210	pass
2211
2212	return value.BashArray(strs) # TODO: Reuse this object too?
2213
2214	elif case('LINENO'):
2215	assert self.token_for_line is not None
2216	# Reuse object with mutation
2217	# TODO: maybe use interned GetLineNumStr?
2218	self.line_num.s = str(self.token_for_line.line.line_num)
2219	return self.line_num
2220
2221	elif case('BASHPID'): # TODO: YSH io->getpid()
2222	return value.Str(str(posix.getpid()))
2223
2224	elif case('_'):
2225	return value.Str(self.last_arg)
2226
2227	elif case('SECONDS'):
2228	f = time_.time() - self.seconds_start
2229	ok, big_int = mops.FromFloat(f)
2230	assert ok, f # should never be NAN or INFINITY
2231	return value.Int(big_int)
2232
2233	else:
2234	# In the case 'declare -n ref='a[42]', the result won't be a cell. Idea to
2235	# fix this:
2236	# 1. Call self.unsafe_arith.ParseVarRef() -> BracedVarSub
2237	# 2. Call self.unsafe_arith.GetNameref(bvs_part), and get a value_t
2238	# We still need a ref_trail to detect cycles.
2239	cell, _, _ = self._ResolveNameOrRef(name, which_scopes)
2240	if cell:
2241	return cell.val
2242
2243	builtin_val = self.builtins.get(name)
2244	if builtin_val:
2245	return builtin_val
2246
2247	# TODO: Can look in the builtins module, which is a value.Obj
2248	return value.Undef
2249
2250	def GetCell(self, name, which_scopes=scope_e.Shopt):
2251	# type: (str, scope_t) -> Cell
2252	"""Get both the value and flags.
2253
2254	Usages:
2255	- the 'pp' builtin.
2256	- declare -p
2257	- ${x@a}
2258	- to test of 'TZ' is exported in printf? Why?
2259
2260	Note: consulting __builtins__ doesn't see necessary for any of these
2261	"""
2262	if which_scopes == scope_e.Shopt:
2263	which_scopes = self.ScopesForReading()
2264
2265	cell, _ = self._ResolveNameOnly(name, which_scopes)
2266	return cell
2267
2268	def Unset(self, lval, which_scopes):
2269	# type: (sh_lvalue_t, scope_t) -> bool
2270	"""
2271	Returns:
2272	Whether the cell was found.
2273	"""
2274	# TODO: Refactor sh_lvalue type to avoid this
2275	UP_lval = lval
2276
2277	with tagswitch(lval) as case:
2278	if case(sh_lvalue_e.Var): # unset x
2279	lval = cast(LeftName, UP_lval)
2280	var_name = lval.name
2281	elif case(sh_lvalue_e.Indexed): # unset 'a[1]'
2282	lval = cast(sh_lvalue.Indexed, UP_lval)
2283	var_name = lval.name
2284	elif case(sh_lvalue_e.Keyed): # unset 'A["K"]'
2285	lval = cast(sh_lvalue.Keyed, UP_lval)
2286	var_name = lval.name
2287	else:
2288	raise AssertionError()
2289
2290	if which_scopes == scope_e.Shopt:
2291	which_scopes = self.ScopesForWriting()
2292
2293	cell, var_frame, cell_name = self._ResolveNameOrRef(
2294	var_name, which_scopes)
2295	if not cell:
2296	return False # 'unset' builtin falls back on functions
2297	if cell.readonly:
2298	raise error.Runtime("Can't unset readonly variable %r" % var_name)
2299
2300	with tagswitch(lval) as case:
2301	if case(sh_lvalue_e.Var): # unset x
2302	# Make variables in higher scopes visible.
2303	# example: test/spec.sh builtin-vars -r 24 (ble.sh)
2304	mylib.dict_erase(var_frame, cell_name)
2305
2306	# alternative that some shells use:
2307	# var_frame[cell_name].val = value.Undef
2308	# cell.exported = False
2309
2310	# This should never happen because we do recursive lookups of namerefs.
2311	assert not cell.nameref, cell
2312
2313	elif case(sh_lvalue_e.Indexed): # unset 'a[1]'
2314	lval = cast(sh_lvalue.Indexed, UP_lval)
2315	# Note: Setting an entry to None and shifting entries are pretty
2316	# much the same in shell.
2317
2318	val = cell.val
2319	UP_val = val
2320	if val.tag() == value_e.BashArray:
2321	val = cast(value.BashArray, UP_val)
2322	error_code = bash_impl.BashArray_UnsetElement(
2323	val, lval.index)
2324	if error_code == error_code_e.IndexOutOfRange:
2325	n = bash_impl.BashArray_Length(val)
2326	raise error.Runtime(
2327	"%s[%d]: Index is out of bounds for array of length %d"
2328	% (var_name, lval.index, n))
2329	elif val.tag() == value_e.SparseArray:
2330	val = cast(value.SparseArray, UP_val)
2331	error_code = bash_impl.SparseArray_UnsetElement(
2332	val, mops.IntWiden(lval.index))
2333	if error_code == error_code_e.IndexOutOfRange:
2334	big_length = bash_impl.SparseArray_Length(val)
2335	raise error.Runtime(
2336	"%s[%d]: Index is out of bounds for array of length %s"
2337	% (var_name, lval.index, mops.ToStr(big_length)))
2338	else:
2339	raise error.Runtime("%r isn't an array" % var_name)
2340
2341	elif case(sh_lvalue_e.Keyed): # unset 'A["K"]'
2342	lval = cast(sh_lvalue.Keyed, UP_lval)
2343
2344	val = cell.val
2345	UP_val = val
2346
2347	# note: never happens because of mem.IsBashAssoc test for sh_lvalue.Keyed
2348	#if val.tag() != value_e.BashAssoc:
2349	# raise error.Runtime("%r isn't an associative array" % lval.name)
2350
2351	val = cast(value.BashAssoc, UP_val)
2352	bash_impl.BashAssoc_UnsetElement(val, lval.key)
2353
2354	else:
2355	raise AssertionError(lval)
2356
2357	return True
2358
2359	def ScopesForReading(self):
2360	# type: () -> scope_t
2361	"""Read scope."""
2362	return (scope_e.Dynamic
2363	if self.exec_opts.dynamic_scope() else scope_e.LocalOrGlobal)
2364
2365	def ScopesForWriting(self):
2366	# type: () -> scope_t
2367	"""Write scope."""
2368	return (scope_e.Dynamic
2369	if self.exec_opts.dynamic_scope() else scope_e.LocalOnly)
2370
2371	def ClearFlag(self, name, flag):
2372	# type: (str, int) -> bool
2373	"""Used for export -n.
2374
2375	We don't use SetValue() because even if rval is None, it will make an
2376	Undef value in a scope.
2377	"""
2378	cell, var_frame = self._ResolveNameOnly(name, self.ScopesForReading())
2379	if cell:
2380	if flag & ClearExport:
2381	cell.exported = False
2382	if flag & ClearNameref:
2383	cell.nameref = False
2384	return True
2385	else:
2386	return False
2387
2388	def _FillWithExported(self, new_env):
2389	# type: (Dict[str, str]) -> None
2390
2391	# Search from globals up. Names higher on the stack will overwrite
2392	# names lower on the stack.
2393	for scope in self.var_stack:
2394	for name, cell in iteritems(scope):
2395	if cell.exported and cell.val.tag() == value_e.Str:
2396	val = cast(value.Str, cell.val)
2397	new_env[name] = val.s
2398
2399	def _FillEnvObj(self, new_env, env_object):
2400	# type: (Dict[str, str], Obj) -> None
2401
2402	# Do the LEAST visible parts first
2403	if env_object.prototype is not None:
2404	self._FillEnvObj(new_env, env_object.prototype)
2405
2406	# Overwrite with MOST visible parts
2407	for name, val in iteritems(env_object.d):
2408	if val.tag() != value_e.Str:
2409	continue
2410	new_env[name] = cast(value.Str, val).s
2411
2412	def GetEnv(self):
2413	# type: () -> Dict[str, str]
2414	"""
2415	Get the environment that should be used for launching processes.
2416
2417	Note: This is run on every SimpleCommand. Should we have a dirty
2418	flag? We could notice these things:
2419
2420	- If an exported variable is changed
2421	- If the set of exported variables changes.
2422	"""
2423	new_env = {} # type: Dict[str, str]
2424
2425	# Note: ysh:upgrade has both of these behaviors
2426
2427	# OSH: Consult exported vars
2428	if not self.exec_opts.no_exported():
2429	self._FillWithExported(new_env)
2430
2431	# YSH: Consult the ENV dict
2432	if self.exec_opts.env_obj():
2433	self._FillEnvObj(new_env, self.env_object)
2434
2435	return new_env
2436
2437	def VarNames(self):
2438	# type: () -> List[str]
2439	"""For internal OSH completion and compgen -A variable.
2440
2441	NOTE: We could also add $? $$ etc.?
2442	"""
2443	ret = [] # type: List[str]
2444	# Look up the stack, yielding all variables. Bash seems to do this.
2445	for scope in self.var_stack:
2446	for name in scope:
2447	ret.append(name)
2448	return ret
2449
2450	def VarNamesStartingWith(self, prefix):
2451	# type: (str) -> List[str]
2452	"""For ${!prefix@}"""
2453	# Look up the stack, yielding all variables. Bash seems to do this.
2454	names = [] # type: List[str]
2455	for scope in self.var_stack:
2456	for name in scope:
2457	if name.startswith(prefix):
2458	names.append(name)
2459	return names
2460
2461	def GetAllVars(self):
2462	# type: () -> Dict[str, str]
2463	"""Get all variables and their values, for 'set' builtin."""
2464	result = {} # type: Dict[str, str]
2465	for scope in self.var_stack:
2466	for name, cell in iteritems(scope):
2467	# TODO: Show other types?
2468	val = cell.val
2469	if val.tag() == value_e.Str:
2470	str_val = cast(value.Str, val)
2471	result[name] = str_val.s
2472	return result
2473
2474	def GetAllCells(self, which_scopes):
2475	# type: (scope_t) -> Dict[str, Cell]
2476	"""Get all variables and their values, for 'set' builtin."""
2477	result = {} # type: Dict[str, Cell]
2478
2479	if which_scopes == scope_e.Dynamic:
2480	scopes = self.var_stack
2481	elif which_scopes == scope_e.LocalOnly:
2482	scopes = self.var_stack[-1:]
2483	elif which_scopes == scope_e.GlobalOnly:
2484	scopes = self.var_stack[0:1]
2485	elif which_scopes == scope_e.LocalOrGlobal:
2486	scopes = [self.var_stack[0]]
2487	if len(self.var_stack) > 1:
2488	scopes.append(self.var_stack[-1])
2489	else:
2490	raise AssertionError()
2491
2492	for scope in scopes:
2493	for name, cell in iteritems(scope):
2494	result[name] = cell
2495	return result
2496
2497	def SetRegexMatch(self, match):
2498	# type: (regex_match_t) -> None
2499	self.regex_match[-1] = match
2500
2501	def GetRegexMatch(self):
2502	# type: () -> regex_match_t
2503	return self.regex_match[-1]
2504
2505	def PushContextStack(self, context):
2506	# type: (Dict[str, value_t]) -> None
2507	self.ctx_stack.append(context)
2508
2509	def GetContext(self):
2510	# type: () -> Optional[Dict[str, value_t]]
2511	if len(self.ctx_stack):
2512	return self.ctx_stack[-1]
2513	return None
2514
2515	def PopContextStack(self):
2516	# type: () -> Dict[str, value_t]
2517	assert self.ctx_stack, "Empty context stack"
2518	return self.ctx_stack.pop()
2519
2520
2521	def ValueIsInvokableObj(val):
2522	# type: (value_t) -> Tuple[Optional[value_t], Optional[Obj]]
2523	"""
2524	Returns:
2525	(__invoke__ Proc or BuiltinProc, self Obj) if the value is invokable
2526	(None, None) otherwise
2527	"""
2528	if val.tag() != value_e.Obj:
2529	return None, None
2530
2531	obj = cast(Obj, val)
2532	if not obj.prototype:
2533	return None, None
2534
2535	invoke_val = obj.prototype.d.get('__invoke__')
2536	if invoke_val is None:
2537	return None, None
2538
2539	# TODO: __invoke__ of wrong type could be fatal error?
2540	if invoke_val.tag() in (value_e.Proc, value_e.BuiltinProc):
2541	return invoke_val, obj
2542
2543	return None, None
2544
2545
2546	def _AddNames(unique, frame):
2547	# type: (Dict[str, bool], Dict[str, Cell]) -> None
2548	for name in frame:
2549	val = frame[name].val
2550	if val.tag() == value_e.Proc:
2551	unique[name] = True
2552	proc, _ = ValueIsInvokableObj(val)
2553	if proc is not None:
2554	unique[name] = True
2555
2556
2557	class Procs(object):
2558	"""
2559	Terminology:
2560
2561	- invokable - these are INTERIOR
2562	- value.Proc - which can be shell function in __sh_funcs__ namespace, or
2563	YSH proc
2564	- value.Obj with __invoke__
2565	- exterior - external commands, extern builtin
2566
2567	Note: the YSH 'invoke' builtin can generalize YSH 'runproc' builtin, shell command/builtin,
2568	and also type / type -a
2569	"""
2570
2571	def __init__(self, mem):
2572	# type: (Mem) -> None
2573	self.mem = mem
2574	self.sh_funcs = {} # type: Dict[str, value.Proc]
2575
2576	def DefineShellFunc(self, name, proc):
2577	# type: (str, value.Proc) -> None
2578	self.sh_funcs[name] = proc
2579
2580	def IsShellFunc(self, name):
2581	# type: (str) -> bool
2582	return name in self.sh_funcs
2583
2584	def GetShellFunc(self, name):
2585	# type: (str) -> Optional[value.Proc]
2586	return self.sh_funcs.get(name)
2587
2588	def EraseShellFunc(self, to_del):
2589	# type: (str) -> None
2590	"""Undefine a sh-func with name `to_del`, if it exists."""
2591	mylib.dict_erase(self.sh_funcs, to_del)
2592
2593	def ShellFuncNames(self):
2594	# type: () -> List[str]
2595	"""Returns a sorted list of all shell function names
2596
2597	Callers:
2598	declare -f -F
2599	"""
2600	names = self.sh_funcs.keys()
2601	names.sort()
2602	return names
2603
2604	def DefineProc(self, name, proc):
2605	# type: (str, value.Proc) -> None
2606	"""
2607	procs are defined in the local scope.
2608	"""
2609	self.mem.var_stack[-1][name] = Cell(False, False, False, proc)
2610
2611	def IsProc(self, name):
2612	# type: (str) -> bool
2613
2614	maybe_proc = self.mem.GetValue(name)
2615	# Could be Undef
2616	return maybe_proc.tag() == value_e.Proc
2617
2618	def IsInvokableObj(self, name):
2619	# type: (str) -> bool
2620
2621	val = self.mem.GetValue(name)
2622	proc, _ = ValueIsInvokableObj(val)
2623	return proc is not None
2624
2625	def InvokableNames(self):
2626	# type: () -> List[str]
2627	"""Returns a sorted list of all invokable names
2628
2629	Callers:
2630	complete -A function
2631	pp proc - should deprecate this
2632	"""
2633	unique = NewDict() # type: Dict[str, bool]
2634	for name in self.sh_funcs:
2635	unique[name] = True
2636
2637	top_frame = self.mem.var_stack[-1]
2638	_AddNames(unique, top_frame)
2639
2640	global_frame = self.mem.var_stack[0]
2641	#log('%d %d', id(top_frame), id(global_frame))
2642	if global_frame is not top_frame:
2643	_AddNames(unique, global_frame)
2644
2645	#log('%s', unique)
2646
2647	names = unique.keys()
2648	names.sort()
2649
2650	return names
2651
2652	def GetInvokable(self, name):
2653	# type: (str) -> Tuple[Optional[value_t], Optional[Obj]]
2654	"""Find a proc, invokable Obj, or sh-func, in that order
2655
2656	Callers:
2657	executor.py: to actually run
2658	meta_oils.py runproc lookup - this is not 'invoke', because it is
2659	INTERIOR shell functions, procs, invokable Obj
2660	"""
2661	val = self.mem.GetValue(name)
2662
2663	if val.tag() == value_e.Proc:
2664	return cast(value.Proc, val), None
2665
2666	proc, self_val = ValueIsInvokableObj(val)
2667	if proc:
2668	return proc, self_val
2669
2670	if name in self.sh_funcs:
2671	return self.sh_funcs[name], None
2672
2673	return None, None
2674
2675
2676	#
2677	# Wrappers to Set Variables
2678	#
2679
2680
2681	def OshLanguageSetValue(mem, lval, val, flags=0):
2682	# type: (Mem, sh_lvalue_t, value_t, int) -> None
2683	"""Like 'setvar' (scope_e.LocalOnly), unless dynamic scope is on.
2684
2685	That is, it respects shopt --unset dynamic_scope.
2686
2687	Used for assignment builtins, (( a = b )), {fd}>out, ${x=}, etc.
2688	"""
2689	which_scopes = mem.ScopesForWriting()
2690	mem.SetValue(lval, val, which_scopes, flags=flags)
2691
2692
2693	def BuiltinSetValue(mem, lval, val):
2694	# type: (Mem, sh_lvalue_t, value_t) -> None
2695	"""Equivalent of x=$y
2696
2697	Called by BuiltinSetString and BuiltinSetArray Used directly by
2698	printf -v because it can mutate an array
2699	"""
2700	mem.SetValue(lval, val, mem.ScopesForWriting())
2701
2702
2703	def BuiltinSetString(mem, name, s):
2704	# type: (Mem, str, str) -> None
2705	"""Set a string by looking up the stack.
2706
2707	Used for 'read', 'getopts', completion builtins, etc.
2708	"""
2709	assert isinstance(s, str)
2710	BuiltinSetValue(mem, location.LName(name), value.Str(s))
2711
2712
2713	def BuiltinSetArray(mem, name, a):
2714	# type: (Mem, str, List[str]) -> None
2715	"""Set an array by looking up the stack.
2716
2717	Used by compadjust, read -a, etc.
2718	"""
2719	assert isinstance(a, list)
2720	BuiltinSetValue(mem, location.LName(name), value.BashArray(a))
2721
2722
2723	def SetGlobalString(mem, name, s):
2724	# type: (Mem, str, str) -> None
2725	"""Helper for completion, etc."""
2726	assert isinstance(s, str)
2727	val = value.Str(s)
2728	mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly)
2729
2730
2731	def SetGlobalArray(mem, name, a):
2732	# type: (Mem, str, List[str]) -> None
2733	"""Used by completion, shell initialization, etc."""
2734	assert isinstance(a, list)
2735	mem.SetNamed(location.LName(name), value.BashArray(a), scope_e.GlobalOnly)
2736
2737
2738	def SetGlobalValue(mem, name, val):
2739	# type: (Mem, str, value_t) -> None
2740	"""Helper for completion, etc."""
2741	mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly)
2742
2743
2744	def SetLocalValue(mem, name, val):
2745	# type: (Mem, str, value_t) -> None
2746	"""For 'use' builtin."""
2747	mem.SetNamed(location.LName(name), val, scope_e.LocalOnly)
2748
2749
2750	def ExportGlobalString(mem, name, s):
2751	# type: (Mem, str, str) -> None
2752	"""Helper for completion, $PWD, $OLDPWD, etc."""
2753	assert isinstance(s, str)
2754	val = value.Str(s)
2755	mem.SetNamed(location.LName(name),
2756	val,
2757	scope_e.GlobalOnly,
2758	flags=SetExport)
2759
2760
2761	# TODO: remove in favor of EnvConfig
2762	def SetStringInEnv(mem, var_name, s):
2763	# type: (Mem, str, str) -> None
2764	if mem.exec_opts.env_obj(): # e.g. ENV.YSH_HISTFILE
2765	mem.env_dict[var_name] = value.Str(s)
2766	else: # e.g. $YSH_HISTFILE
2767	SetGlobalString(mem, var_name, s)
2768
2769
2770	#
2771	# Wrappers to Get Variables
2772	#
2773
2774
2775	def DynamicGetVar(mem, name, which_scopes):
2776	# type: (Mem, str, scope_t) -> value_t
2777	"""
2778	For getVar() and shvarGet()
2779	"""
2780	val = mem.GetValue(name, which_scopes=which_scopes)
2781
2782	# Undef is not a user-visible value!
2783	# There's no way to distinguish null from undefined.
2784	if val.tag() == value_e.Undef:
2785	return value.Null
2786
2787	return val
2788
2789
2790	def GetString(mem, name):
2791	# type: (Mem, str) -> str
2792	"""Wrapper around GetValue().
2793
2794	Check that HOME, PWD, OLDPWD, etc. are strings. bash doesn't have these
2795	errors because ${array} is ${array[0]}.
2796
2797	TODO: We could also check this when you're storing variables?
2798	"""
2799	val = mem.GetValue(name)
2800	UP_val = val
2801	with tagswitch(val) as case:
2802	if case(value_e.Undef):
2803	raise error.Runtime("$%s isn't defined" % name)
2804	elif case(value_e.Str):
2805	return cast(value.Str, UP_val).s
2806	else:
2807	# User would have to 'unset HOME' to get rid of exported flag
2808	raise error.Runtime("$%s should be a string" % name)
2809
2810
2811	def MaybeString(mem, name):
2812	# type: (Mem, str) -> Optional[str]
2813	"""Like GetString(), but doesn't throw an exception."""
2814	try:
2815	return GetString(mem, name)
2816	except error.Runtime:
2817	return None
2818
2819
2820	def GetInteger(mem, name):
2821	# type: (Mem, str) -> int
2822	"""For OPTIND variable used in getopts builtin.
2823
2824	TODO: it could be value.Int() ?
2825	"""
2826	val = mem.GetValue(name)
2827	if val.tag() != value_e.Str:
2828	raise error.Runtime('$%s should be a string, got %s' %
2829	(name, ui.ValType(val)))
2830	s = cast(value.Str, val).s
2831	try:
2832	i = int(s)
2833	except ValueError:
2834	raise error.Runtime("$%s doesn't look like an integer, got %r" %
2835	(name, s))
2836	return i
2837
2838
2839	# vim: sw=4