core/state.py

OILS / core / state.py View on Github | oils.pub

2837 lines, 1451 significant

1	# Copyright 2016 Andy Chu. All rights reserved.
2	# Licensed under the Apache License, Version 2.0 (the "License");
3	# you may not use this file except in compliance with the License.
4	# You may obtain a copy of the License at
5	#
6	# http://www.apache.org/licenses/LICENSE-2.0
7	"""
8	state.py - Interpreter state
9	"""
10	from __future__ import print_function
11	import time as time_ # avoid name conflict
12
13	from _devbuild.gen.id_kind_asdl import Id
14	from _devbuild.gen.option_asdl import option_i
15	from _devbuild.gen.runtime_asdl import (error_code_e, scope_e, scope_t, Cell)
16	from _devbuild.gen.syntax_asdl import (loc, loc_t, Token, debug_frame,
17	debug_frame_e, debug_frame_t)
18	from _devbuild.gen.types_asdl import opt_group_i
19	from _devbuild.gen.value_asdl import (value, value_e, value_t, Obj, sh_lvalue,
20	sh_lvalue_e, sh_lvalue_t, LeftName,
21	y_lvalue_e, regex_match, regex_match_e,
22	regex_match_t, RegexMatch)
23	from core import bash_impl
24	from core import error
25	from core.error import e_usage, e_die
26	from core import num
27	from core import optview
28	from display import ui
29	from core import util
30	from frontend import consts
31	from frontend import location
32	from frontend import match
33	from mycpp import mops
34	from mycpp import mylib
35	from mycpp.mylib import (log, print_stderr, str_switch, tagswitch, iteritems,
36	NewDict)
37	from pylib import os_path
38
39	from libc import HAVE_GLOB_PERIOD
40	import posix_ as posix
41
42	from typing import Tuple, List, Dict, Optional, Any, cast, TYPE_CHECKING
43
44	if TYPE_CHECKING:
45	from _devbuild.gen.option_asdl import option_t
46	from core import alloc
47	from osh import sh_expr_eval
48
49	_ = log
50
51	# flags for mem.SetValue()
52	SetReadOnly = 1 << 0
53	ClearReadOnly = 1 << 1
54	SetExport = 1 << 2
55	ClearExport = 1 << 3
56	SetNameref = 1 << 4
57	ClearNameref = 1 << 5
58
59
60	class ctx_Source(object):
61	"""For source builtin."""
62
63	def __init__(self, mem, source_name, argv):
64	# type: (Mem, str, List[str]) -> None
65	mem.PushSource(source_name, argv)
66	self.mem = mem
67	self.argv = argv
68
69	# Whenever we're sourcing, the 'is-main' builtin will return 1 (false)
70	self.to_restore = self.mem.is_main
71	self.mem.is_main = False
72
73	def __enter__(self):
74	# type: () -> None
75	pass
76
77	def __exit__(self, type, value, traceback):
78	# type: (Any, Any, Any) -> None
79	self.mem.PopSource(self.argv)
80
81	self.mem.is_main = self.to_restore
82
83
84	class ctx_DebugTrap(object):
85	"""For trap DEBUG."""
86
87	def __init__(self, mem):
88	# type: (Mem) -> None
89	mem.running_debug_trap = True
90	self.mem = mem
91
92	def __enter__(self):
93	# type: () -> None
94	pass
95
96	def __exit__(self, type, value, traceback):
97	# type: (Any, Any, Any) -> None
98	self.mem.running_debug_trap = False
99
100
101	class ctx_ErrTrap(object):
102	"""For trap ERR."""
103
104	def __init__(self, mem):
105	# type: (Mem) -> None
106	mem.running_err_trap = True
107	self.mem = mem
108
109	def __enter__(self):
110	# type: () -> None
111	pass
112
113	def __exit__(self, type, value, traceback):
114	# type: (Any, Any, Any) -> None
115	self.mem.running_err_trap = False
116
117
118	class ctx_Option(object):
119	"""Shopt --unset errexit { false }"""
120
121	def __init__(self, mutable_opts, opt_nums, b):
122	# type: (MutableOpts, List[int], bool) -> None
123	for opt_num in opt_nums:
124	mutable_opts.Push(opt_num, b)
125	if opt_num == option_i.errexit:
126	# it wasn't disabled
127	mutable_opts.errexit_disabled_tok.append(None)
128
129	self.mutable_opts = mutable_opts
130	self.opt_nums = opt_nums
131
132	def __enter__(self):
133	# type: () -> None
134	pass
135
136	def __exit__(self, type, value, traceback):
137	# type: (Any, Any, Any) -> None
138	for opt_num in self.opt_nums: # don't bother to do it in reverse order
139	if opt_num == option_i.errexit:
140	self.mutable_opts.errexit_disabled_tok.pop()
141	self.mutable_opts.Pop(opt_num)
142
143
144	class ctx_AssignBuiltin(object):
145	"""Local x=$(false) is disallowed."""
146
147	def __init__(self, mutable_opts):
148	# type: (MutableOpts) -> None
149	self.strict = False
150	if mutable_opts.Get(option_i.strict_errexit):
151	mutable_opts.Push(option_i._allow_command_sub, False)
152	mutable_opts.Push(option_i._allow_process_sub, False)
153	self.strict = True
154
155	self.mutable_opts = mutable_opts
156
157	def __enter__(self):
158	# type: () -> None
159	pass
160
161	def __exit__(self, type, value, traceback):
162	# type: (Any, Any, Any) -> None
163	if self.strict:
164	self.mutable_opts.Pop(option_i._allow_command_sub)
165	self.mutable_opts.Pop(option_i._allow_process_sub)
166
167
168	class ctx_YshExpr(object):
169	"""Command sub must fail in 'mystring' ++ $(false)"""
170
171	def __init__(self, mutable_opts):
172	# type: (MutableOpts) -> None
173
174	# Similar to $LIB_OSH/bash-strict.sh
175
176	# TODO: consider errexit:all group, or even ysh:all
177	# It would be nice if this were more efficient
178	mutable_opts.Push(option_i.command_sub_errexit, True)
179	mutable_opts.Push(option_i.errexit, True)
180	mutable_opts.Push(option_i.pipefail, True)
181	mutable_opts.Push(option_i.inherit_errexit, True)
182	mutable_opts.Push(option_i.strict_errexit, True)
183
184	# What about nounset? This has a similar pitfall -- it's not running
185	# like YSH.
186	# e.g. var x = $(echo $zz)
187
188	self.mutable_opts = mutable_opts
189
190	def __enter__(self):
191	# type: () -> None
192	pass
193
194	def __exit__(self, type, value, traceback):
195	# type: (Any, Any, Any) -> None
196	self.mutable_opts.Pop(option_i.command_sub_errexit)
197	self.mutable_opts.Pop(option_i.errexit)
198	self.mutable_opts.Pop(option_i.pipefail)
199	self.mutable_opts.Pop(option_i.inherit_errexit)
200	self.mutable_opts.Pop(option_i.strict_errexit)
201
202
203	class ctx_ErrExit(object):
204	"""Manages the errexit setting.
205
206	- The user can change it with builtin 'set' at any point in the code.
207	- These constructs implicitly disable 'errexit':
208	- if / while / until conditions
209	- ! (part of pipeline)
210	- && \|\|
211	"""
212
213	def __init__(self, mutable_opts, b, disabled_tok):
214	# type: (MutableOpts, bool, Optional[Token]) -> None
215
216	# If we're disabling it, we need a span ID. If not, then we should NOT
217	# have one.
218	assert b == (disabled_tok is None)
219
220	mutable_opts.Push(option_i.errexit, b)
221	mutable_opts.errexit_disabled_tok.append(disabled_tok)
222
223	self.strict = False
224	if mutable_opts.Get(option_i.strict_errexit):
225	mutable_opts.Push(option_i._allow_command_sub, False)
226	mutable_opts.Push(option_i._allow_process_sub, False)
227	self.strict = True
228
229	self.mutable_opts = mutable_opts
230
231	def __enter__(self):
232	# type: () -> None
233	pass
234
235	def __exit__(self, type, value, traceback):
236	# type: (Any, Any, Any) -> None
237	self.mutable_opts.errexit_disabled_tok.pop()
238	self.mutable_opts.Pop(option_i.errexit)
239
240	if self.strict:
241	self.mutable_opts.Pop(option_i._allow_command_sub)
242	self.mutable_opts.Pop(option_i._allow_process_sub)
243
244
245	class OptHook(object):
246	"""Interface for option hooks."""
247
248	def __init__(self):
249	# type: () -> None
250	"""Empty constructor for mycpp."""
251	pass
252
253	def OnChange(self, opt0_array, opt_name, b):
254	# type: (List[bool], str, bool) -> bool
255	"""This method is called whenever an option is changed.
256
257	Returns success or failure.
258	"""
259	return True
260
261
262	def InitOpts():
263	# type: () -> List[bool]
264
265	opt0_array = [False] * option_i.ARRAY_SIZE
266	for opt_num in consts.DEFAULT_TRUE:
267	opt0_array[opt_num] = True
268	return opt0_array
269
270
271	def MakeOpts(
272	mem, # type: Mem
273	environ, # type: Dict[str, str]
274	opt_hook, # type: OptHook
275	):
276	# type: (...) -> Tuple[optview.Parse, optview.Exec, MutableOpts]
277
278	# Unusual representation: opt0_array + opt_stacks. For two features:
279	#
280	# - POSIX errexit disable semantics
281	# - YSH shopt --set nullglob { ... }
282	#
283	# We could do it with a single List of stacks. But because shopt --set
284	# random_option { ... } is very uncommon, we optimize and store the ZERO
285	# element of the stack in a flat array opt0_array (default False), and then
286	# the rest in opt_stacks, where the value could be None. By allowing the
287	# None value, we save ~50 or so list objects in the common case.
288
289	opt0_array = InitOpts()
290	# Overrides, including errexit
291	no_stack = None # type: List[bool] # for mycpp
292	opt_stacks = [no_stack] * option_i.ARRAY_SIZE # type: List[List[bool]]
293
294	parse_opts = optview.Parse(opt0_array, opt_stacks)
295	exec_opts = optview.Exec(opt0_array, opt_stacks)
296	mutable_opts = MutableOpts(mem, environ, opt0_array, opt_stacks, opt_hook)
297
298	return parse_opts, exec_opts, mutable_opts
299
300
301	def _SetGroup(opt0_array, opt_nums, b):
302	# type: (List[bool], List[int], bool) -> None
303	for opt_num in opt_nums:
304	b2 = not b if opt_num in consts.DEFAULT_TRUE else b
305	opt0_array[opt_num] = b2
306
307
308	def MakeYshParseOpts():
309	# type: () -> optview.Parse
310	opt0_array = InitOpts()
311	_SetGroup(opt0_array, consts.YSH_ALL, True)
312
313	no_stack = None # type: List[bool]
314	opt_stacks = [no_stack] * option_i.ARRAY_SIZE # type: List[List[bool]]
315
316	parse_opts = optview.Parse(opt0_array, opt_stacks)
317	return parse_opts
318
319
320	def _AnyOptionNum(opt_name, ignore_shopt_not_impl):
321	# type: (str, bool) -> option_t
322	opt_num = consts.OptionNum(opt_name)
323	if opt_num == 0:
324	if ignore_shopt_not_impl:
325	opt_num = consts.UnimplOptionNum(opt_name)
326	if opt_num == 0:
327	e_usage('got invalid option %r' % opt_name, loc.Missing)
328
329	# Note: we relaxed this for YSH so we can do 'shopt --unset errexit' consistently
330	#if opt_num not in consts.SHOPT_OPTION_NUMS:
331	# e_usage("doesn't own option %r (try 'set')" % opt_name)
332
333	return opt_num
334
335
336	def _SetOptionNum(opt_name):
337	# type: (str) -> option_t
338	opt_num = consts.OptionNum(opt_name)
339	if opt_num == 0:
340	e_usage('got invalid option %r' % opt_name, loc.Missing)
341
342	if opt_num not in consts.SET_OPTION_NUMS:
343	e_usage("invalid option %r (try shopt)" % opt_name, loc.Missing)
344
345	return opt_num
346
347
348	def _MaybeWarnDotglob():
349	# type: () -> None
350	if HAVE_GLOB_PERIOD == 0:
351	# GNU libc and musl libc have GLOB_PERIOD, but Android doesn't
352	print_stderr(
353	"osh warning: GLOB_PERIOD wasn't found in libc, so 'shopt -s dotglob' won't work"
354	)
355
356
357	class MutableOpts(object):
358
359	def __init__(self, mem, environ, opt0_array, opt_stacks, opt_hook):
360	# type: (Mem, Dict[str, str], List[bool], List[List[bool]], OptHook) -> None
361	self.mem = mem
362	self.environ = environ
363	self.opt0_array = opt0_array
364	self.opt_stacks = opt_stacks
365	self.errexit_disabled_tok = [] # type: List[Token]
366
367	# Used for 'set -o vi/emacs'
368	self.opt_hook = opt_hook
369
370	def Init(self):
371	# type: () -> None
372
373	# This comes after all the 'set' options.
374	shellopts = self.mem.GetValue('SHELLOPTS')
375
376	# True in OSH, but not in YSH (no_init_globals)
377	if shellopts.tag() == value_e.Str:
378	s = cast(value.Str, shellopts).s
379	self._InitOptionsFromEnv(s)
380
381	def _InitOptionsFromEnv(self, shellopts):
382	# type: (str) -> None
383	# e.g. errexit:nounset:pipefail
384	lookup = shellopts.split(':')
385	for opt_num in consts.SET_OPTION_NUMS:
386	name = consts.OptionName(opt_num)
387	if name in lookup:
388	self._SetOldOption(name, True)
389
390	def Push(self, opt_num, b):
391	# type: (int, bool) -> None
392	if opt_num == option_i.dotglob:
393	_MaybeWarnDotglob()
394
395	overlay = self.opt_stacks[opt_num]
396	if overlay is None or len(overlay) == 0:
397	self.opt_stacks[opt_num] = [b] # Allocate a new list
398	else:
399	overlay.append(b)
400
401	def Pop(self, opt_num):
402	# type: (int) -> bool
403	overlay = self.opt_stacks[opt_num]
404	assert overlay is not None
405	return overlay.pop()
406
407	def PushDynamicScope(self, b):
408	# type: (bool) -> None
409	"""B: False if it's a proc, and True if it's a shell function."""
410	# If it's already disabled, keep it disabled
411	if not self.Get(option_i.dynamic_scope):
412	b = False
413	self.Push(option_i.dynamic_scope, b)
414
415	def PopDynamicScope(self):
416	# type: () -> None
417	self.Pop(option_i.dynamic_scope)
418
419	def Get(self, opt_num):
420	# type: (int) -> bool
421	# Like _Getter in core/optview.py
422	overlay = self.opt_stacks[opt_num]
423	if overlay is None or len(overlay) == 0:
424	return self.opt0_array[opt_num]
425	else:
426	return overlay[-1] # the top value
427
428	def _Set(self, opt_num, b):
429	# type: (int, bool) -> None
430	"""Used to disable errexit.
431
432	For bash compatibility in command sub.
433	"""
434	if opt_num == option_i.dotglob:
435	_MaybeWarnDotglob()
436
437	# Like _Getter in core/optview.py
438	overlay = self.opt_stacks[opt_num]
439	if overlay is None or len(overlay) == 0:
440	self.opt0_array[opt_num] = b
441	else:
442	overlay[-1] = b # The top value
443
444	def set_interactive(self):
445	# type: () -> None
446	self._Set(option_i.interactive, True)
447
448	def set_redefine_const(self):
449	# type: () -> None
450	"""For interactive shells."""
451	self._Set(option_i.redefine_const, True)
452
453	def set_redefine_source(self):
454	# type: () -> None
455	"""For interactive shells. For source-guard"""
456	self._Set(option_i.redefine_source, True)
457
458	def set_emacs(self):
459	# type: () -> None
460	self._Set(option_i.emacs, True)
461
462	def _SetArrayByNum(self, opt_num, b):
463	# type: (int, bool) -> None
464	if (opt_num in consts.PARSE_OPTION_NUMS and
465	not self.mem.ParsingChangesAllowed()):
466	e_die('Syntax options must be set at the top level '
467	'(outside any function)')
468
469	self._Set(opt_num, b)
470
471	def SetDeferredErrExit(self, b):
472	# type: (bool) -> None
473	"""Set the errexit flag, possibly deferring it.
474
475	Implements the unusual POSIX "defer" behavior. Callers: set -o
476	errexit, shopt -s ysh:all, ysh:upgrade
477	"""
478	#log('Set %s', b)
479
480	# Defer it until we pop by setting the BOTTOM OF THE STACK.
481	self.opt0_array[option_i.errexit] = b
482
483	def DisableErrExit(self):
484	# type: () -> None
485	"""Called by core/process.py to implement bash quirks."""
486	self._Set(option_i.errexit, False)
487
488	def ErrExitDisabledToken(self):
489	# type: () -> Optional[Token]
490	"""If errexit is disabled by POSIX rules, return Token for construct.
491
492	e.g. the Token for 'if' or '&&' etc.
493	"""
494	# Bug fix: The errexit disabling inherently follows a STACK DISCIPLINE.
495	# But we run trap handlers in the MAIN LOOP, which break this. So just
496	# declare that it's never disabled in a trap.
497	if self.Get(option_i._running_trap):
498	return None
499
500	if len(self.errexit_disabled_tok) == 0:
501	return None
502
503	return self.errexit_disabled_tok[-1]
504
505	def ErrExitIsDisabled(self):
506	# type: () -> bool
507	"""
508	Similar to ErrExitDisabledToken, for ERR trap
509	"""
510	if len(self.errexit_disabled_tok) == 0:
511	return False
512
513	return self.errexit_disabled_tok[-1] is not None
514
515	def _SetOldOption(self, opt_name, b):
516	# type: (str, bool) -> None
517	"""Private version for synchronizing from SHELLOPTS."""
518	assert '_' not in opt_name
519	assert opt_name in consts.SET_OPTION_NAMES
520
521	opt_num = consts.OptionNum(opt_name)
522	assert opt_num != 0, opt_name
523
524	if opt_num == option_i.errexit:
525	self.SetDeferredErrExit(b)
526	else:
527	if opt_num == option_i.verbose and b:
528	print_stderr('osh warning: set -o verbose not implemented')
529	self._SetArrayByNum(opt_num, b)
530
531	# note: may FAIL before we get here.
532
533	success = self.opt_hook.OnChange(self.opt0_array, opt_name, b)
534
535	def SetOldOption(self, opt_name, b):
536	# type: (str, bool) -> None
537	"""For set -o, set +o, or shopt -s/-u -o."""
538	unused = _SetOptionNum(opt_name) # validate it
539	self._SetOldOption(opt_name, b)
540
541	if not self.Get(option_i.no_init_globals):
542	UP_val = self.mem.GetValue('SHELLOPTS')
543	assert UP_val.tag() == value_e.Str, UP_val
544	val = cast(value.Str, UP_val)
545	shellopts = val.s
546
547	# Now check if SHELLOPTS needs to be updated. It may be exported.
548	#
549	# NOTE: It might be better to skip rewriting SEHLLOPTS in the common case
550	# where it is not used. We could do it lazily upon GET.
551
552	# Also, it would be slightly more efficient to update SHELLOPTS if
553	# settings were batched, Examples:
554	# - set -eu
555	# - shopt -s foo bar
556	if b:
557	if opt_name not in shellopts:
558	new_val = value.Str('%s:%s' % (shellopts, opt_name))
559	self.mem.InternalSetGlobal('SHELLOPTS', new_val)
560	else:
561	if opt_name in shellopts:
562	names = [n for n in shellopts.split(':') if n != opt_name]
563	new_val = value.Str(':'.join(names))
564	self.mem.InternalSetGlobal('SHELLOPTS', new_val)
565
566	def SetAnyOption(self, opt_name, b, ignore_shopt_not_impl=False):
567	# type: (str, bool, bool) -> None
568	"""For shopt -s/-u and sh -O/+O."""
569
570	# shopt -s ysh:all turns on all YSH options, which includes all strict
571	# options
572	opt_group = consts.OptionGroupNum(opt_name)
573	if opt_group == opt_group_i.YshUpgrade:
574	_SetGroup(self.opt0_array, consts.YSH_UPGRADE, b)
575	self.SetDeferredErrExit(b) # Special case
576	if b: # ENV dict
577	self.mem.MaybeInitEnvDict(self.environ)
578	return
579
580	if opt_group == opt_group_i.YshAll:
581	_SetGroup(self.opt0_array, consts.YSH_ALL, b)
582	self.SetDeferredErrExit(b) # Special case
583	if b: # ENV dict
584	self.mem.MaybeInitEnvDict(self.environ)
585	return
586
587	if opt_group == opt_group_i.StrictAll:
588	_SetGroup(self.opt0_array, consts.STRICT_ALL, b)
589	return
590
591	opt_num = _AnyOptionNum(opt_name, ignore_shopt_not_impl)
592
593	if opt_num == option_i.errexit:
594	self.SetDeferredErrExit(b)
595	return
596
597	self._SetArrayByNum(opt_num, b)
598
599
600	class _ArgFrame(object):
601	"""Stack frame for arguments array."""
602
603	def __init__(self, argv):
604	# type: (List[str]) -> None
605	self.argv = argv
606	self.num_shifted = 0
607
608	def __repr__(self):
609	# type: () -> str
610	return '<_ArgFrame %s %d at %x>' % (self.argv, self.num_shifted,
611	id(self))
612
613	def Dump(self):
614	# type: () -> Dict[str, value_t]
615	items = [value.Str(s) for s in self.argv] # type: List[value_t]
616	argv = value.List(items)
617	return {
618	'argv': argv,
619	'num_shifted': num.ToBig(self.num_shifted),
620	}
621
622	def GetArgNum(self, arg_num):
623	# type: (int) -> value_t
624
625	# $0 is handled elsewhere
626	assert 1 <= arg_num, arg_num
627
628	index = self.num_shifted + arg_num - 1
629	if index >= len(self.argv):
630	return value.Undef
631
632	return value.Str(self.argv[index])
633
634	def GetArgv(self):
635	# type: () -> List[str]
636	return self.argv[self.num_shifted:]
637
638	def GetNumArgs(self):
639	# type: () -> int
640	return len(self.argv) - self.num_shifted
641
642	def SetArgv(self, argv):
643	# type: (List[str]) -> None
644	self.argv = argv
645	self.num_shifted = 0
646
647
648	def _DumpVarFrame(frame):
649	# type: (Dict[str, Cell]) -> Dict[str, value_t]
650	"""Dump the stack frame as reasonably compact and readable JSON."""
651
652	vars_json = {} # type: Dict[str, value_t]
653	for name, cell in iteritems(frame):
654	cell_json = {} # type: Dict[str, value_t]
655
656	buf = mylib.BufWriter()
657	if cell.exported:
658	buf.write('x')
659	if cell.readonly:
660	buf.write('r')
661	flags = buf.getvalue()
662	if len(flags):
663	cell_json['flags'] = value.Str(flags)
664
665	# TODO:
666	# - Use packle for crash dumps! Then we can represent object cycles
667	# - Right now the JSON serializer will probably crash
668	# - although BashArray and BashAssoc may need 'type' tags
669	# - they don't round trip correctly
670	# - maybe add value.Tombstone here or something?
671	# - value.{Func,Eggex,...} may have value.Tombstone and
672	# vm.ValueIdString()?
673
674	with tagswitch(cell.val) as case:
675	if case(value_e.Undef):
676	cell_json['val'] = value.Null
677
678	elif case(value_e.Str, value_e.BashArray, value_e.BashAssoc,
679	value_e.SparseArray):
680	cell_json['val'] = cell.val
681
682	else:
683	# TODO: should we show the object ID here?
684	pass
685
686	vars_json[name] = value.Dict(cell_json)
687
688	return vars_json
689
690
691	def _LineNumber(tok):
692	# type: (Optional[Token]) -> str
693	""" For $BASH_LINENO """
694	if tok is None:
695	return '-1'
696	return str(tok.line.line_num)
697
698
699	def _AddCallToken(d, token):
700	# type: (Dict[str, value_t], Optional[Token]) -> None
701	if token is None:
702	return
703	d['call_source'] = value.Str(ui.GetLineSourceString(token.line))
704	d['call_line_num'] = num.ToBig(token.line.line_num)
705	d['call_line'] = value.Str(token.line.content)
706
707
708	class ctx_FuncCall(object):
709	"""For func calls."""
710
711	def __init__(self, mem, func):
712	# type: (Mem, value.Func) -> None
713
714	self.saved_globals = mem.var_stack[0]
715
716	assert func.module_frame is not None
717	mem.var_stack[0] = func.module_frame
718
719	frame = NewDict() # type: Dict[str, Cell]
720	mem.var_stack.append(frame)
721
722	mem.PushCall(func.name, func.parsed.name)
723
724	self.mem = mem
725
726	def __enter__(self):
727	# type: () -> None
728	pass
729
730	def __exit__(self, type, value, traceback):
731	# type: (Any, Any, Any) -> None
732	self.mem.PopCall()
733	self.mem.var_stack.pop()
734
735	self.mem.var_stack[0] = self.saved_globals
736
737
738	class ctx_ProcCall(object):
739	"""For proc calls, including shell functions."""
740
741	def __init__(self, mem, mutable_opts, proc, argv):
742	# type: (Mem, MutableOpts, value.Proc, List[str]) -> None
743
744	# TODO:
745	# should we separate procs and shell functions?
746	# - dynamic scope is one difference
747	# - '$@" shift etc. are another difference
748
749	self.saved_globals = mem.var_stack[0]
750
751	assert proc.module_frame is not None
752	mem.var_stack[0] = proc.module_frame
753
754	frame = NewDict() # type: Dict[str, Cell]
755
756	assert argv is not None
757	if proc.sh_compat:
758	# shell function
759	mem.argv_stack.append(_ArgFrame(argv))
760	else:
761	# procs
762	# - open: is equivalent to ...ARGV
763	# - closed: ARGV is empty list
764	frame['ARGV'] = _MakeArgvCell(argv)
765
766	mem.var_stack.append(frame)
767
768	mem.PushCall(proc.name, proc.name_tok)
769
770	# Dynamic scope is only for shell functions
771	mutable_opts.PushDynamicScope(proc.sh_compat)
772
773	# It may have been disabled with ctx_ErrExit for 'if echo $(false)', but
774	# 'if p' should be allowed.
775	self.mem = mem
776	self.mutable_opts = mutable_opts
777	self.sh_compat = proc.sh_compat
778
779	def __enter__(self):
780	# type: () -> None
781	pass
782
783	def __exit__(self, type, value, traceback):
784	# type: (Any, Any, Any) -> None
785	self.mutable_opts.PopDynamicScope()
786	self.mem.PopCall()
787	self.mem.var_stack.pop()
788
789	if self.sh_compat:
790	self.mem.argv_stack.pop()
791
792	self.mem.var_stack[0] = self.saved_globals
793
794
795	class ctx_Temp(object):
796	""" POSIX shell FOO=bar mycommand """
797
798	def __init__(self, mem):
799	# type: (Mem) -> None
800	self.mem = mem
801	mem.PushTemp()
802
803	def __enter__(self):
804	# type: () -> None
805	pass
806
807	def __exit__(self, type, value, traceback):
808	# type: (Any, Any, Any) -> None
809	self.mem.PopTemp()
810
811
812	class ctx_EnvObj(object):
813	"""YSH FOO=bar my-command"""
814
815	def __init__(self, mem, bindings):
816	# type: (Mem, Dict[str, value_t]) -> None
817	self.mem = mem
818	mem.PushEnvObj(bindings)
819
820	def __enter__(self):
821	# type: () -> None
822	pass
823
824	def __exit__(self, type, value, traceback):
825	# type: (Any, Any, Any) -> None
826	self.mem.PopEnvObj()
827
828
829	class ctx_Registers(object):
830	"""For $PS1, $PS4, $PROMPT_COMMAND, traps, and headless EVAL.
831
832	This is tightly coupled to state.Mem, so it's not in builtin/pure_ysh.
833	"""
834
835	def __init__(self, mem):
836	# type: (Mem) -> None
837
838	# Because some prompts rely on the status leaking. See issue #853.
839	# PS1 also does.
840	last = mem.last_status[-1]
841	mem.last_status.append(last)
842	mem.try_status.append(0)
843	mem.try_error.append(value.Dict({}))
844
845	# TODO: We should also copy these values! Turn the whole thing into a
846	# frame.
847	mem.pipe_status.append([])
848	mem.process_sub_status.append([])
849
850	mem.regex_match.append(regex_match.No)
851
852	self.mem = mem
853
854	def __enter__(self):
855	# type: () -> None
856	pass
857
858	def __exit__(self, type, value, traceback):
859	# type: (Any, Any, Any) -> None
860	self.mem.regex_match.pop()
861
862	self.mem.process_sub_status.pop()
863	self.mem.pipe_status.pop()
864
865	self.mem.try_error.pop()
866	self.mem.try_status.pop()
867	self.mem.last_status.pop()
868
869
870	class ctx_ThisDir(object):
871	"""For $_this_dir."""
872
873	def __init__(self, mem, filename):
874	# type: (Mem, Optional[str]) -> None
875	self.do_pop = False
876	if filename is not None: # script_name in main() may be -c, etc.
877	d = os_path.dirname(os_path.abspath(filename))
878	mem.this_dir.append(d)
879	self.do_pop = True
880
881	self.mem = mem
882
883	def __enter__(self):
884	# type: () -> None
885	pass
886
887	def __exit__(self, type, value, traceback):
888	# type: (Any, Any, Any) -> None
889	if self.do_pop:
890	self.mem.this_dir.pop()
891
892
893	def _MakeArgvCell(argv):
894	# type: (List[str]) -> Cell
895	items = [value.Str(a) for a in argv] # type: List[value_t]
896	return Cell(False, False, False, value.List(items))
897
898
899	class ctx_LoopFrame(object):
900
901	def __init__(self, mem, do_new_frame):
902	# type: (Mem, bool) -> None
903	self.mem = mem
904	self.do_new_frame = do_new_frame
905
906	if self.do_new_frame:
907	to_enclose = self.mem.var_stack[-1]
908	self.new_frame = NewDict() # type: Dict[str, Cell]
909	self.new_frame['__E__'] = Cell(False, False, False,
910	value.Frame(to_enclose))
911	mem.var_stack.append(self.new_frame)
912
913	def __enter__(self):
914	# type: () -> None
915	pass
916
917	def __exit__(self, type, value, traceback):
918	# type: (Any, Any, Any) -> None
919	if self.do_new_frame:
920	self.mem.var_stack.pop()
921
922
923	class ctx_EnclosedFrame(object):
924	"""
925	Usages:
926
927	- io->evalToDict(), which is a primitive used for Hay and the Dict proc
928	- lexical scope aka static scope for block args to user-defined procs
929	- Including the "closures in a loop" problem, which will be used for Hay
930
931	var mutated = 'm'
932	var shadowed = 's'
933
934	Dict (&d) {
935	shadowed = 42
936	mutated = 'new' # this is equivalent to var mutated
937
938	setvar mutated = 'new'
939	}
940	echo $shadowed # restored to 's'
941	echo $mutated # new
942
943	Or maybe we disallow the setvar lookup?
944	"""
945
946	def __init__(
947	self,
948	mem, # type: Mem
949	to_enclose, # type: Dict[str, Cell]
950	module_frame, # type: Dict[str, Cell]
951	out_dict, # type: Optional[Dict[str, value_t]]
952	):
953	# type: (...) -> None
954	self.mem = mem
955	self.to_enclose = to_enclose
956	self.module_frame = module_frame
957	self.out_dict = out_dict
958
959	if module_frame is not None:
960	self.saved_globals = self.mem.var_stack[0]
961	self.mem.var_stack[0] = module_frame
962
963	# __E__ gets a lookup rule
964	self.new_frame = NewDict() # type: Dict[str, Cell]
965	self.new_frame['__E__'] = Cell(False, False, False,
966	value.Frame(to_enclose))
967
968	mem.var_stack.append(self.new_frame)
969
970	def __enter__(self):
971	# type: () -> None
972	pass
973
974	def __exit__(self, type, value, traceback):
975	# type: (Any, Any, Any) -> None
976
977	if self.out_dict is not None:
978	for name, cell in iteritems(self.new_frame):
979	#log('name %r', name)
980	#log('cell %r', cell)
981
982	# User can hide variables with _ suffix
983	# e.g. for i_ in foo bar { echo $i_ }
984	if name.endswith('_'):
985	continue
986
987	self.out_dict[name] = cell.val
988
989	# Restore
990	self.mem.var_stack.pop()
991
992	if self.module_frame is not None:
993	self.mem.var_stack[0] = self.saved_globals
994
995
996	class ctx_ModuleEval(object):
997	"""Evaluate a module with a new global stack frame.
998
999	e.g. setglobal in the new module doesn't leak
1000
1001	Different from ctx_EnclosedFrame because the new code can't see variables in
1002	the old frame.
1003	"""
1004
1005	def __init__(self, mem, out_dict, out_errors):
1006	# type: (Mem, Dict[str, value_t], List[str]) -> None
1007	self.mem = mem
1008	self.out_dict = out_dict
1009	self.out_errors = out_errors
1010
1011	self.new_frame = NewDict() # type: Dict[str, Cell]
1012	self.saved_frame = mem.var_stack[0]
1013
1014	# Somewhat of a hack for tracing within a module.
1015	# Other solutions:
1016	# - PS4 can be __builtin__, but that would break shell compatibility
1017	# - We can have a separate YSH mechanism that uses a different settings
1018	# - We probably still want it to be scoped, like shvar PS4=z { ... }
1019	#
1020	# Note: there's a similar issue with HOSTNAME UID EUID etc. But those
1021	# could be io.hostname() io.getuid(), or lazy constants, etc.
1022
1023	ps4 = self.saved_frame.get('PS4')
1024	if ps4:
1025	self.new_frame['PS4'] = ps4
1026	# ENV is not in __builtins__ because it's mutable -- we want
1027	# 'setglobal' to work
1028	env = self.saved_frame.get('ENV')
1029	if env:
1030	self.new_frame['ENV'] = env
1031
1032	assert len(mem.var_stack) == 1
1033	mem.var_stack[0] = self.new_frame
1034
1035	# Whenever we're use-ing, the 'is-main' builtin will return 1 (false)
1036	self.to_restore = self.mem.is_main
1037	self.mem.is_main = False
1038
1039	def __enter__(self):
1040	# type: () -> None
1041	pass
1042
1043	def __exit__(self, type, value_, traceback):
1044	# type: (Any, Any, Any) -> None
1045
1046	self.mem.is_main = self.to_restore
1047
1048	assert len(self.mem.var_stack) == 1
1049	self.mem.var_stack[0] = self.saved_frame
1050
1051	# Now look in __export__ for the list of names to expose
1052
1053	cell = self.new_frame.get('__provide__')
1054	if cell is None:
1055	self.out_errors.append("Module is missing __provide__ List")
1056	return
1057
1058	provide_val = cell.val
1059	with tagswitch(provide_val) as case:
1060	if case(value_e.List):
1061	for val in cast(value.List, provide_val).items:
1062	if val.tag() == value_e.Str:
1063	name = cast(value.Str, val).s
1064
1065	cell = self.new_frame.get(name)
1066	if cell is None:
1067	self.out_errors.append(
1068	"Name %r was provided, but not defined" % name)
1069	continue
1070
1071	self.out_dict[name] = cell.val
1072	else:
1073	self.out_errors.append(
1074	"Expected Str in __provide__ List, got %s" %
1075	ui.ValType(val))
1076
1077	else:
1078	self.out_errors.append("__provide__ should be a List, got %s" %
1079	ui.ValType(provide_val))
1080
1081
1082	class ctx_Eval(object):
1083	"""Push temporary set of variables, $0, $1, $2, etc."""
1084
1085	def __init__(
1086	self,
1087	mem, # type: Mem
1088	dollar0, # type: Optional[str]
1089	pos_args, # type: Optional[List[str]]
1090	vars, # type: Optional[Dict[str, value_t]]
1091	):
1092	# type: (...) -> None
1093	self.mem = mem
1094	self.dollar0 = dollar0
1095	self.pos_args = pos_args
1096	self.vars = vars
1097
1098	# $0 needs to have lexical scoping. So we store it with other locals.
1099	# As "0" cannot be parsed as an lvalue, we can safely store dollar0 there.
1100	if dollar0 is not None:
1101	#assert mem.GetValue("0", scope_e.LocalOnly).tag() == value_e.Undef
1102	#self.dollar0_lval = LeftName("0", loc.Missing)
1103	#mem.SetLocalName(self.dollar0_lval, value.Str(dollar0))
1104
1105	self.restore_dollar0 = self.mem.dollar0
1106	self.mem.dollar0 = dollar0
1107
1108	if pos_args is not None:
1109	mem.argv_stack.append(_ArgFrame(pos_args))
1110
1111	if vars is not None:
1112	self.restore = [] # type: List[Tuple[LeftName, value_t]]
1113	self._Push(vars)
1114
1115	def __enter__(self):
1116	# type: () -> None
1117	pass
1118
1119	def __exit__(self, type, value_, traceback):
1120	# type: (Any, Any, Any) -> None
1121	if self.vars is not None:
1122	self._Pop()
1123
1124	if self.pos_args is not None:
1125	self.mem.argv_stack.pop()
1126
1127	if self.dollar0 is not None:
1128	#self.mem.SetLocalName(self.dollar0_lval, value.Undef)
1129	self.mem.dollar0 = self.restore_dollar0
1130
1131	# Note: _Push and _Pop are separate methods because the C++ translation
1132	# doesn't like when they are inline in __init__ and __exit__.
1133	def _Push(self, vars):
1134	# type: (Dict[str, value_t]) -> None
1135	for name in vars:
1136	lval = location.LName(name)
1137	# LocalOnly because we are only overwriting the current scope
1138	old_val = self.mem.GetValue(name, scope_e.LocalOnly)
1139	self.restore.append((lval, old_val))
1140	self.mem.SetNamed(lval, vars[name], scope_e.LocalOnly)
1141
1142	def _Pop(self):
1143	# type: () -> None
1144	for lval, old_val in self.restore:
1145	if old_val.tag() == value_e.Undef:
1146	self.mem.Unset(lval, scope_e.LocalOnly)
1147	else:
1148	self.mem.SetNamed(lval, old_val, scope_e.LocalOnly)
1149
1150
1151	def _FrameLookup(frame, name):
1152	# type: (Dict[str, Cell], str) -> Tuple[Optional[Cell], Dict[str, Cell]]
1153	"""
1154	Look for a name in the frame, then recursively into the enclosing __E__
1155	frame, if it exists
1156	"""
1157	cell = frame.get(name)
1158	if cell:
1159	return cell, frame
1160
1161	rear_cell = frame.get('__E__') # ctx_EnclosedFrame() sets this
1162	if rear_cell:
1163	rear_val = rear_cell.val
1164	assert rear_val, rear_val
1165	if rear_val.tag() == value_e.Frame:
1166	to_enclose = cast(value.Frame, rear_val).frame
1167	return _FrameLookup(to_enclose, name) # recursive call
1168
1169	return None, None
1170
1171
1172	class Mem(object):
1173	"""For storing variables.
1174
1175	Callers:
1176	User code: assigning and evaluating variables, in command context or
1177	arithmetic context.
1178	Completion engine: for COMP_WORDS, etc.
1179	Builtins call it implicitly: read, cd for $PWD, $OLDPWD, etc.
1180
1181	Modules: cmd_eval, word_eval, expr_eval, completion
1182	"""
1183
1184	def __init__(
1185	self,
1186	dollar0, # type: str
1187	argv, # type: List[str]
1188	arena, # type: alloc.Arena
1189	debug_stack, # type: List[debug_frame_t]
1190	env_dict, # type: Dict[str, value_t]
1191	defaults=None, # type: Dict[str, value_t]
1192	):
1193	# type: (...) -> None
1194	"""
1195	Args:
1196	arena: currently unused
1197	"""
1198	# circular dep initialized out of line
1199	self.exec_opts = None # type: optview.Exec
1200	self.unsafe_arith = None # type: sh_expr_eval.UnsafeArith
1201
1202	self.dollar0 = dollar0
1203	# If you only use YSH procs and funcs, this will remain at length 1.
1204	self.argv_stack = [_ArgFrame(argv)]
1205
1206	frame = NewDict() # type: Dict[str, Cell]
1207
1208	frame['ARGV'] = _MakeArgvCell(argv)
1209
1210	self.var_stack = [frame]
1211
1212	# The debug_stack isn't strictly necessary for execution. We use it
1213	# for crash dumps and for 3 parallel arrays: BASH_SOURCE, FUNCNAME, and
1214	# BASH_LINENO.
1215	self.debug_stack = debug_stack
1216
1217	self.env_dict = env_dict
1218	self.env_object = Obj(None, env_dict) # initial state
1219
1220	if defaults is None: # for unit tests only
1221	self.defaults = NewDict() # type: Dict[str, value_t]
1222	else:
1223	self.defaults = defaults
1224
1225	self.pwd = None # type: Optional[str]
1226	self.seconds_start = time_.time()
1227
1228	self.token_for_line = None # type: Optional[Token]
1229	self.loc_for_expr = loc.Missing # type: loc_t
1230
1231	self.last_arg = '' # $_ is initially empty, NOT unset
1232	self.line_num = value.Str('')
1233
1234	# Done ONCE on initialization
1235	self.root_pid = posix.getpid()
1236
1237	# TODO:
1238	# - These are REGISTERS mutated by user code.
1239	# - Call it self.reg_stack? with ctx_Registers
1240	# - push-registers builtin
1241	self.last_status = [0] # type: List[int] # a stack
1242	self.try_status = [0] # type: List[int] # a stack
1243	self.try_error = [value.Dict({})] # type: List[value.Dict] # a stack
1244	self.pipe_status = [[]] # type: List[List[int]] # stack
1245	self.process_sub_status = [[]] # type: List[List[int]] # stack
1246
1247	# A stack but NOT a register?
1248	self.this_dir = [] # type: List[str]
1249	self.regex_match = [regex_match.No] # type: List[regex_match_t]
1250
1251	self.last_bg_pid = -1 # Uninitialized value mutable public variable
1252
1253	self.running_debug_trap = False # set by ctx_DebugTrap()
1254	self.running_err_trap = False # set by ctx_ErrTrap
1255	self.is_main = True # we start out in main
1256
1257	# For the ctx builtin
1258	self.ctx_stack = [] # type: List[Dict[str, value_t]]
1259
1260	self.builtins = NewDict() # type: Dict[str, value_t]
1261
1262	# Note: Python 2 and 3 have __builtins__
1263	# This is just for inspection
1264	builtins_module = Obj(None, self.builtins)
1265
1266	# Code in any module can see __builtins__
1267	self.builtins['__builtins__'] = builtins_module
1268
1269	self.did_ysh_env = False # only initialize ENV once per process
1270
1271	from core import sh_init
1272	self.env_config = sh_init.EnvConfig(self, defaults)
1273
1274	def __repr__(self):
1275	# type: () -> str
1276	parts = [] # type: List[str]
1277	parts.append('<Mem')
1278	for i, frame in enumerate(self.var_stack):
1279	parts.append(' -- %d --' % i)
1280	for n, v in frame.iteritems():
1281	parts.append(' %s %s' % (n, v))
1282	parts.append('>')
1283	return '\n'.join(parts) + '\n'
1284
1285	def AddBuiltin(self, name, val):
1286	# type: (str, value_t) -> None
1287	self.builtins[name] = val
1288
1289	def SetPwd(self, pwd):
1290	# type: (str) -> None
1291	"""Used by builtins."""
1292	self.pwd = pwd
1293
1294	def ParsingChangesAllowed(self):
1295	# type: () -> bool
1296	"""For checking that syntax options are only used at the top level."""
1297
1298	# DISALLOW proc calls : they push argv_stack, var_stack, debug_stack
1299	# ALLOW source foo.sh arg1: pushes argv_stack, debug_stack
1300	# ALLOW FOO=bar : pushes var_stack
1301	return len(self.var_stack) == 1 or len(self.argv_stack) == 1
1302
1303	def Dump(self):
1304	# type: () -> Tuple[List[value_t], List[value_t], List[value_t]]
1305	"""Copy state before unwinding the stack."""
1306	var_stack = [
1307	value.Dict(_DumpVarFrame(frame)) for frame in self.var_stack
1308	] # type: List[value_t]
1309	argv_stack = [value.Dict(frame.Dump())
1310	for frame in self.argv_stack] # type: List[value_t]
1311
1312	debug_stack = [] # type: List[value_t]
1313
1314	# Reuse these immutable objects
1315	t_call = value.Str('Call')
1316	t_source = value.Str('Source')
1317	t_main = value.Str('MainFile')
1318
1319	for frame in reversed(self.debug_stack):
1320	UP_frame = frame
1321	with tagswitch(frame) as case:
1322	if case(debug_frame_e.Call):
1323	frame = cast(debug_frame.Call, UP_frame)
1324	d = {
1325	'type': t_call,
1326	'func_name': value.Str(frame.func_name)
1327	} # type: Dict[str, value_t]
1328
1329	_AddCallToken(d, frame.call_tok)
1330	# TODO: Add def_tok
1331
1332	elif case(debug_frame_e.Source):
1333	frame = cast(debug_frame.Source, UP_frame)
1334	d = {
1335	'type': t_source,
1336	'source_name': value.Str(frame.source_name)
1337	}
1338	_AddCallToken(d, frame.call_tok)
1339
1340	elif case(debug_frame_e.MainFile):
1341	frame = cast(debug_frame.MainFile, UP_frame)
1342	d = {
1343	'type': t_main,
1344	'filename': value.Str(frame.main_filename)
1345	}
1346
1347	debug_stack.append(value.Dict(d))
1348	return var_stack, argv_stack, debug_stack
1349
1350	def SetLastArgument(self, s):
1351	# type: (str) -> None
1352	"""For $_"""
1353	self.last_arg = s
1354
1355	def SetTokenForLine(self, tok):
1356	# type: (Token) -> None
1357	"""Set a token to compute $LINENO
1358
1359	This means it should be set on SimpleCommand, ShAssignment, ((, [[,
1360	case, etc. -- anything that evaluates a word. Example: there was a bug
1361	with 'case $LINENO'
1362
1363	This token also used as a "least-specific" / fallback location for
1364	errors in ExecuteAndCatch().
1365
1366	Although most of that should be taken over by 'with ui.ctx_Location()`,
1367	for the errfmt.
1368	"""
1369	if self.running_debug_trap or self.running_err_trap:
1370	return
1371
1372	#if tok.span_id == runtime.NO_SPID:
1373	# NOTE: This happened in the osh-runtime benchmark for yash.
1374	#log('Warning: span_id undefined in SetTokenForLine')
1375
1376	#import traceback
1377	#traceback.print_stack()
1378	#return
1379
1380	# Reset the expression fallback location on every line. It REFINES the
1381	# line-based fallback location.
1382	self.loc_for_expr = loc.Missing
1383
1384	self.token_for_line = tok
1385
1386	def SetLocationForExpr(self, blame_loc):
1387	# type: (loc_t) -> None
1388	"""
1389	A more specific fallback location, like the $[ in
1390
1391	echo $[len(42)]
1392	"""
1393	self.loc_for_expr = blame_loc
1394
1395	def GetFallbackLocation(self):
1396	# type: () -> loc_t
1397
1398	if self.loc_for_expr != loc.Missing: # more specific
1399	return self.loc_for_expr
1400
1401	if self.token_for_line: # less specific
1402	return self.token_for_line
1403
1404	return loc.Missing
1405
1406	#
1407	# Status Variable Stack (for isolating $PS1 and $PS4)
1408	#
1409
1410	def LastStatus(self):
1411	# type: () -> int
1412	return self.last_status[-1]
1413
1414	def TryStatus(self):
1415	# type: () -> int
1416	return self.try_status[-1]
1417
1418	def TryError(self):
1419	# type: () -> value.Dict
1420	return self.try_error[-1]
1421
1422	def PipeStatus(self):
1423	# type: () -> List[int]
1424	return self.pipe_status[-1]
1425
1426	def SetLastStatus(self, x):
1427	# type: (int) -> None
1428	self.last_status[-1] = x
1429
1430	def SetTryStatus(self, x):
1431	# type: (int) -> None
1432	self.try_status[-1] = x
1433
1434	def SetTryError(self, x):
1435	# type: (value.Dict) -> None
1436	self.try_error[-1] = x
1437
1438	def SetPipeStatus(self, x):
1439	# type: (List[int]) -> None
1440	self.pipe_status[-1] = x
1441
1442	def SetSimplePipeStatus(self, status):
1443	# type: (int) -> None
1444
1445	# Optimization to avoid allocations
1446	top = self.pipe_status[-1]
1447	if len(top) == 1:
1448	top[0] = status
1449	else:
1450	self.pipe_status[-1] = [status]
1451
1452	def SetProcessSubStatus(self, x):
1453	# type: (List[int]) -> None
1454	self.process_sub_status[-1] = x
1455
1456	#
1457	# Call Stack
1458	#
1459
1460	def PushCall(self, func_name, def_tok):
1461	# type: (str, Token) -> None
1462	"""Push argv, var, and debug stack frames.
1463
1464	Currently used for proc and func calls. TODO: New func evaluator may
1465	not use it.
1466
1467	Args:
1468	def_tok: Token where proc or func was defined, used to compute
1469	BASH_SOURCE.
1470	"""
1471	self.debug_stack.append(
1472	debug_frame.Call(self.token_for_line, def_tok, func_name))
1473
1474	def PopCall(self):
1475	# type: () -> None
1476	"""
1477	Args:
1478	should_pop_argv_stack: Pass False if PushCall was given None for argv
1479	True for proc, False for func
1480	"""
1481	self.debug_stack.pop()
1482
1483	def ShouldRunDebugTrap(self):
1484	# type: () -> bool
1485
1486	# TODO: RunLastPart of pipeline can disable this
1487
1488	# Don't recursively run DEBUG trap
1489	if self.running_debug_trap:
1490	return False
1491
1492	# Don't run it inside functions
1493	if len(self.var_stack) > 1:
1494	return False
1495
1496	return True
1497
1498	def IsGlobalScope(self):
1499	# type: () -> bool
1500	"""
1501	local -g uses this, probably because bash does the wrong thing and
1502	prints LOCALS, not globals.
1503	"""
1504	return len(self.var_stack) == 1
1505
1506	def InsideFunction(self):
1507	# type: () -> bool
1508	"""For the ERR trap, and use builtin"""
1509
1510	# TODO: Should this be unified with ParsingChangesAllowed()? Slightly
1511	# different logic.
1512
1513	# Don't run it inside functions
1514	return len(self.var_stack) > 1
1515
1516	def GlobalFrame(self):
1517	# type: () -> Dict[str, Cell]
1518	"""For defining the global scope of modules.
1519
1520	It's affected by ctx_ModuleEval()
1521	"""
1522	return self.var_stack[0]
1523
1524	def CurrentFrame(self):
1525	# type: () -> Dict[str, Cell]
1526	"""For attaching a stack frame to a value.Block"""
1527	return self.var_stack[-1]
1528
1529	def PushSource(self, source_name, argv):
1530	# type: (str, List[str]) -> None
1531	""" For 'source foo.sh 1 2 3' """
1532	if len(argv):
1533	self.argv_stack.append(_ArgFrame(argv))
1534
1535	# self.token_for_line can be None?
1536	self.debug_stack.append(
1537	debug_frame.Source(self.token_for_line, source_name))
1538
1539	def PopSource(self, argv):
1540	# type: (List[str]) -> None
1541	self.debug_stack.pop()
1542
1543	if len(argv):
1544	self.argv_stack.pop()
1545
1546	def PushTemp(self):
1547	# type: () -> None
1548	"""For the temporary scope in 'FOO=bar BAR=baz echo'.
1549
1550	Also for PS4 evaluation with more variables.
1551	"""
1552	# We don't want the 'read' builtin to write to this frame!
1553	frame = NewDict() # type: Dict[str, Cell]
1554	self.var_stack.append(frame)
1555
1556	def PopTemp(self):
1557	# type: () -> None
1558	self.var_stack.pop()
1559
1560	def _BindEnvObj(self):
1561	# type: () -> None
1562	self.SetNamed(location.LName('ENV'), self.env_object,
1563	scope_e.GlobalOnly)
1564
1565	def MaybeInitEnvDict(self, environ):
1566	# type: (Dict[str, str]) -> None
1567	if self.did_ysh_env:
1568	return
1569
1570	for name, s in iteritems(environ):
1571	self.env_dict[name] = value.Str(s)
1572
1573	self._BindEnvObj()
1574	self.did_ysh_env = True
1575
1576	def PushEnvObj(self, bindings):
1577	# type: (Dict[str, value_t]) -> None
1578	"""Push "bindings" as the MOST visible part of the ENV Obj
1579
1580	i.e. first() / propView()
1581	"""
1582	self.env_object = Obj(self.env_object, bindings)
1583	self._BindEnvObj()
1584
1585	def PopEnvObj(self):
1586	# type: () -> None
1587	"""Pop a Dict of bindings."""
1588	self.env_object = self.env_object.prototype
1589	if self.env_object is None:
1590	# Note: there isn't a way to hit this now, but let's be defensive.
1591	# See test case in spec/ysh-env.test.sh.
1592	e_die('PopEnvObj: env.prototype is null', loc.Missing)
1593
1594	self._BindEnvObj()
1595
1596	#
1597	# Argv
1598	#
1599
1600	def Shift(self, n):
1601	# type: (int) -> int
1602	frame = self.argv_stack[-1]
1603	num_args = len(frame.argv)
1604
1605	if (frame.num_shifted + n) <= num_args:
1606	frame.num_shifted += n
1607	return 0 # success
1608	else:
1609	return 1 # silent error
1610
1611	def GetArg0(self):
1612	# type: () -> value.Str
1613	"""Like GetArgNum(0) but with a more specific type."""
1614	return value.Str(self.dollar0)
1615
1616	def GetArgNum(self, arg_num):
1617	# type: (int) -> value_t
1618	if arg_num == 0:
1619	# Disabled
1620	if 0:
1621	# Problem: Doesn't obey enclosing frame?
1622	# Yeah it needs FrameLookup
1623	cell, _ = _FrameLookup(self.var_stack[-1], '0')
1624	if cell is not None:
1625	val = cell.val
1626	if val.tag() != value_e.Undef:
1627	return val
1628
1629	return value.Str(self.dollar0)
1630
1631	return self.argv_stack[-1].GetArgNum(arg_num)
1632
1633	def GetArgv(self):
1634	# type: () -> List[str]
1635	"""For $* and $@."""
1636	return self.argv_stack[-1].GetArgv()
1637
1638	def SetArgv(self, argv):
1639	# type: (List[str]) -> None
1640	"""For set -- 1 2 3."""
1641	# from set -- 1 2 3
1642	self.argv_stack[-1].SetArgv(argv)
1643
1644	#
1645	# Special Vars
1646	#
1647
1648	def GetSpecialVar(self, op_id):
1649	# type: (int) -> value_t
1650	if op_id == Id.VSub_Bang: # $!
1651	n = self.last_bg_pid
1652	if n == -1:
1653	return value.Undef # could be an error
1654
1655	elif op_id == Id.VSub_QMark: # $?
1656	# External commands need WIFEXITED test. What about subshells?
1657	n = self.last_status[-1]
1658
1659	elif op_id == Id.VSub_Pound: # $#
1660	n = self.argv_stack[-1].GetNumArgs()
1661
1662	elif op_id == Id.VSub_Dollar: # $$
1663	n = self.root_pid
1664
1665	else:
1666	raise NotImplementedError(op_id)
1667
1668	return value.Str(str(n))
1669
1670	#
1671	# Named Vars
1672	#
1673
1674	def _ResolveNameOnly(self, name, which_scopes):
1675	# type: (str, scope_t) -> Tuple[Optional[Cell], Dict[str, Cell]]
1676	"""Helper for getting and setting variable.
1677
1678	Returns:
1679	cell: The cell corresponding to looking up 'name' with the given mode, or
1680	None if it's not found.
1681	var_frame: The frame it should be set to or deleted from.
1682	"""
1683	if which_scopes == scope_e.Dynamic:
1684	for i in xrange(len(self.var_stack) - 1, -1, -1):
1685	var_frame = self.var_stack[i]
1686	cell, result_frame = _FrameLookup(var_frame, name)
1687	if cell:
1688	return cell, result_frame
1689	return None, self.var_stack[0] # set in global var_frame
1690
1691	if which_scopes == scope_e.LocalOnly:
1692	var_frame = self.var_stack[-1]
1693	cell, result_frame = _FrameLookup(var_frame, name)
1694	if cell:
1695	return cell, result_frame
1696	return None, var_frame
1697
1698	if which_scopes == scope_e.GlobalOnly:
1699	var_frame = self.var_stack[0]
1700	cell, result_frame = _FrameLookup(var_frame, name)
1701	if cell:
1702	return cell, result_frame
1703
1704	return None, var_frame
1705
1706	if which_scopes == scope_e.LocalOrGlobal:
1707	# Local
1708	var_frame = self.var_stack[-1]
1709	cell, result_frame = _FrameLookup(var_frame, name)
1710	if cell:
1711	return cell, result_frame
1712
1713	# Global
1714	var_frame = self.var_stack[0]
1715	cell, result_frame = _FrameLookup(var_frame, name)
1716	if cell:
1717	return cell, result_frame
1718
1719	return None, var_frame
1720
1721	raise AssertionError()
1722
1723	def _ResolveNameOrRef(
1724	self,
1725	name, # type: str
1726	which_scopes, # type: scope_t
1727	ref_trail=None, # type: Optional[List[str]]
1728	):
1729	# type: (...) -> Tuple[Optional[Cell], Dict[str, Cell], str]
1730	"""Look up a cell and namespace, but respect the nameref flag.
1731
1732	Resolving namerefs does RECURSIVE calls.
1733	"""
1734	cell, var_frame = self._ResolveNameOnly(name, which_scopes)
1735
1736	if cell is None or not cell.nameref:
1737	return cell, var_frame, name # not a nameref
1738
1739	val = cell.val
1740	UP_val = val
1741	with tagswitch(val) as case:
1742	if case(value_e.Undef):
1743	# This is 'local -n undef_ref', which is kind of useless, because the
1744	# more common idiom is 'local -n ref=$1'. Note that you can mutate
1745	# references themselves with local -n ref=new.
1746	if self.exec_opts.strict_nameref():
1747	e_die('nameref %r is undefined' % name)
1748	else:
1749	return cell, var_frame, name # fallback
1750
1751	elif case(value_e.Str):
1752	val = cast(value.Str, UP_val)
1753	new_name = val.s
1754
1755	else:
1756	# SetValue() protects the invariant that nameref is Undef or Str
1757	raise AssertionError(val.tag())
1758
1759	# TODO: Respect eval_unsafe_arith here (issue 881). See how it's done in
1760	# 'printf -v' with MakeArithParser
1761	if not match.IsValidVarName(new_name):
1762	# e.g. '#' or '1' or ''
1763	if self.exec_opts.strict_nameref():
1764	e_die('nameref %r contains invalid variable name %r' %
1765	(name, new_name))
1766	else:
1767	# Bash has this odd behavior of clearing the nameref bit when
1768	# ref=#invalid#. strict_nameref avoids it.
1769	cell.nameref = False
1770	return cell, var_frame, name # fallback
1771
1772	# Check for circular namerefs.
1773	if ref_trail is None:
1774	ref_trail = [name]
1775	else:
1776	if new_name in ref_trail:
1777	e_die('Circular nameref %s' % ' -> '.join(ref_trail))
1778	ref_trail.append(new_name)
1779
1780	# 'declare -n' uses dynamic scope.
1781	cell, var_frame, cell_name = self._ResolveNameOrRef(
1782	new_name, scope_e.Dynamic, ref_trail=ref_trail)
1783	return cell, var_frame, cell_name
1784
1785	def IsBashAssoc(self, name):
1786	# type: (str) -> bool
1787	"""Returns whether a name resolve to a cell with an associative array.
1788
1789	We need to know this to evaluate the index expression properly
1790	-- should it be coerced to an integer or not?
1791	"""
1792	cell, _, _ = self._ResolveNameOrRef(name, self.ScopesForReading())
1793	# foo=([key]=value)
1794	return cell is not None and cell.val.tag() == value_e.BashAssoc
1795
1796	def SetPlace(self, place, val, blame_loc):
1797	# type: (value.Place, value_t, loc_t) -> None
1798
1799	yval = place.lval
1800	UP_yval = yval
1801	with tagswitch(yval) as case:
1802	if case(y_lvalue_e.Local):
1803	yval = cast(LeftName, UP_yval)
1804
1805	if 0:
1806	# Check that the frame is still alive
1807	# Note: Disabled because it doesn't work with modules. the
1808	# Place captures a frame in def-test.ysh, which we want to
1809	# mutate while Dict is executing in the module_frame for
1810	# def.ysh. See ctx_ModuleEval
1811	found = False
1812	for i in xrange(len(self.var_stack) - 1, -1, -1):
1813	frame = self.var_stack[i]
1814	if frame is place.frame:
1815	found = True
1816	#log('FOUND %s', found)
1817	break
1818	if not found:
1819	e_die(
1820	"Can't assign to place that's no longer on the call stack.",
1821	blame_loc)
1822
1823	frame = place.frame
1824	cell = frame.get(yval.name)
1825	if cell is None:
1826	cell = Cell(False, False, False, val)
1827	frame[yval.name] = cell
1828	else:
1829	cell.val = val
1830
1831	elif case(y_lvalue_e.Container):
1832	e_die('Container place not implemented', blame_loc)
1833
1834	else:
1835	raise AssertionError()
1836
1837	def SetLocalName(self, lval, val):
1838	# type: (LeftName, value_t) -> None
1839	"""
1840	Set a name in the local scope - used for func/proc param binding, etc.
1841	"""
1842
1843	# Equivalent to
1844	# self._ResolveNameOnly(lval.name, scope_e.LocalOnly)
1845	var_frame = self.var_stack[-1]
1846	cell = var_frame.get(lval.name)
1847
1848	if cell:
1849	if cell.readonly:
1850	e_die("Can't assign to readonly value %r" % lval.name,
1851	lval.blame_loc)
1852	cell.val = val # Mutate value_t
1853	else:
1854	cell = Cell(False, False, False, val)
1855	var_frame[lval.name] = cell
1856
1857	def SetNamed(self, lval, val, which_scopes, flags=0):
1858	# type: (LeftName, value_t, scope_t, int) -> None
1859	if flags & SetNameref or flags & ClearNameref:
1860	# declare -n ref=x # refers to the ref itself
1861	cell, var_frame = self._ResolveNameOnly(lval.name, which_scopes)
1862	cell_name = lval.name
1863	else:
1864	# ref=x # mutates THROUGH the reference
1865
1866	# Note on how to implement declare -n ref='a[42]'
1867	# 1. Call _ResolveNameOnly()
1868	# 2. If cell.nameref, call self.unsafe_arith.ParseVarRef() ->
1869	# BracedVarSub
1870	# 3. Turn BracedVarSub into an sh_lvalue, and call
1871	# self.unsafe_arith.SetValue() wrapper with ref_trail
1872	cell, var_frame, cell_name = self._ResolveNameOrRef(
1873	lval.name, which_scopes)
1874
1875	if cell:
1876	# Clear before checking readonly bit.
1877	# NOTE: Could be cell.flags &= flag_clear_mask
1878	if flags & ClearExport:
1879	cell.exported = False
1880	if flags & ClearReadOnly:
1881	cell.readonly = False
1882	if flags & ClearNameref:
1883	cell.nameref = False
1884
1885	if val is not None: # e.g. declare -rx existing
1886	# Note: this DYNAMIC check means we can't have 'const' in a loop.
1887	# But that's true for 'readonly' too, and hoisting it makes more
1888	# sense anyway.
1889	if cell.readonly:
1890	e_die("Can't assign to readonly value %r" % lval.name,
1891	lval.blame_loc)
1892	cell.val = val # CHANGE VAL
1893
1894	# NOTE: Could be cell.flags \|= flag_set_mask
1895	if flags & SetExport:
1896	cell.exported = True
1897	if flags & SetReadOnly:
1898	cell.readonly = True
1899	if flags & SetNameref:
1900	cell.nameref = True
1901
1902	else:
1903	if val is None: # declare -rx nonexistent
1904	# set -o nounset; local foo; echo $foo # It's still undefined!
1905	val = value.Undef # export foo, readonly foo
1906
1907	cell = Cell(bool(flags & SetExport), bool(flags & SetReadOnly),
1908	bool(flags & SetNameref), val)
1909	var_frame[cell_name] = cell
1910
1911	# Maintain invariant that only strings and undefined cells can be
1912	# exported.
1913	assert cell.val is not None, cell
1914
1915	if cell.val.tag() not in (value_e.Undef, value_e.Str):
1916	if cell.exported:
1917	if self.exec_opts.strict_array():
1918	e_die("Only strings can be exported (strict_array)",
1919	lval.blame_loc)
1920	if cell.nameref:
1921	e_die("nameref must be a string", lval.blame_loc)
1922
1923	def SetValue(self, lval, val, which_scopes, flags=0):
1924	# type: (sh_lvalue_t, value_t, scope_t, int) -> None
1925	"""
1926	Args:
1927	lval: sh_lvalue
1928	val: value, or None if only changing flags
1929	which_scopes:
1930	Local \| Global \| Dynamic - for builtins, PWD, etc.
1931	flags: packed pair (keyword_id, bit mask of set/clear flags)
1932
1933	Note: in bash, PWD=/ changes the directory. But not in dash.
1934	"""
1935	# STRICTNESS / SANENESS:
1936	#
1937	# 1) Don't create arrays automatically, e.g. a[1000]=x
1938	# 2) Never change types? yeah I think that's a good idea, at least for YSH
1939	# (not sh, for compatibility). set -o strict_types or something. That
1940	# means arrays have to be initialized with let arr = [], which is fine.
1941	# This helps with stuff like IFS. It starts off as a string, and assigning
1942	# it to a list is an error. I guess you will have to turn this no for
1943	# bash?
1944	#
1945	# TODO:
1946	# - COMPUTED vars can't be set
1947	# - What about PWD / OLDPWD / UID / EUID ? You can simply make them
1948	# readonly.
1949	# - Maybe PARSE $PS1 and $PS4 when they're set, to avoid the error on use?
1950	# - Other validity: $HOME could be checked for existence
1951
1952	UP_lval = lval
1953	with tagswitch(lval) as case:
1954	if case(sh_lvalue_e.Var):
1955	lval = cast(LeftName, UP_lval)
1956
1957	self.SetNamed(lval, val, which_scopes, flags=flags)
1958
1959	elif case(sh_lvalue_e.Indexed):
1960	lval = cast(sh_lvalue.Indexed, UP_lval)
1961
1962	# There is no syntax 'declare a[x]'
1963	assert val is not None, val
1964
1965	# TODO: relax this for YSH
1966	assert val.tag() == value_e.Str, val
1967	rval = cast(value.Str, val)
1968
1969	# Note: location could be a[x]=1 or (( a[ x ] = 1 ))
1970	left_loc = lval.blame_loc
1971
1972	# bash/mksh have annoying behavior of letting you do LHS assignment to
1973	# Undef, which then turns into an INDEXED array. (Undef means that set
1974	# -o nounset fails.)
1975	cell, var_frame, _ = self._ResolveNameOrRef(
1976	lval.name, which_scopes)
1977	if not cell:
1978	self._BindNewArrayWithEntry(var_frame, lval, rval, flags)
1979	return
1980
1981	if cell.readonly:
1982	e_die("Can't assign to readonly array", left_loc)
1983
1984	UP_cell_val = cell.val
1985	# undef[0]=y is allowed
1986	with tagswitch(UP_cell_val) as case2:
1987	if case2(value_e.Undef):
1988	self._BindNewArrayWithEntry(var_frame, lval, rval,
1989	flags)
1990	return
1991
1992	elif case2(value_e.Str):
1993	# s=x
1994	# s[1]=y # invalid
1995	e_die("Can't assign to items in a string", left_loc)
1996
1997	elif case2(value_e.BashArray):
1998	cell_val = cast(value.BashArray, UP_cell_val)
1999	error_code = bash_impl.BashArray_SetElement(
2000	cell_val, lval.index, rval.s)
2001	if error_code == error_code_e.IndexOutOfRange:
2002	n = bash_impl.BashArray_Length(cell_val)
2003	e_die(
2004	"Index %d is out of bounds for array of length %d"
2005	% (lval.index, n), left_loc)
2006	return
2007
2008	elif case2(value_e.SparseArray):
2009	lhs_sp = cast(value.SparseArray, UP_cell_val)
2010	error_code = bash_impl.SparseArray_SetElement(
2011	lhs_sp, mops.IntWiden(lval.index), rval.s)
2012	if error_code == error_code_e.IndexOutOfRange:
2013	n_big = bash_impl.SparseArray_Length(lhs_sp)
2014	e_die(
2015	"Index %d is out of bounds for array of length %s"
2016	% (lval.index, mops.ToStr(n_big)), left_loc)
2017	return
2018
2019	# This could be an object, eggex object, etc. It won't be
2020	# BashAssoc shouldn because we query IsBashAssoc before evaluating
2021	# sh_lhs. Could conslidate with s[i] case above
2022	e_die(
2023	"Value of type %s can't be indexed" % ui.ValType(cell.val),
2024	left_loc)
2025
2026	elif case(sh_lvalue_e.Keyed):
2027	lval = cast(sh_lvalue.Keyed, UP_lval)
2028
2029	# There is no syntax 'declare A["x"]'
2030	assert val is not None, val
2031	assert val.tag() == value_e.Str, val
2032	rval = cast(value.Str, val)
2033
2034	left_loc = lval.blame_loc
2035
2036	cell, var_frame, _ = self._ResolveNameOrRef(
2037	lval.name, which_scopes)
2038	if cell.readonly:
2039	e_die("Can't assign to readonly associative array",
2040	left_loc)
2041
2042	# We already looked it up before making the sh_lvalue
2043	assert cell.val.tag() == value_e.BashAssoc, cell
2044	cell_val2 = cast(value.BashAssoc, cell.val)
2045	bash_impl.BashAssoc_SetElement(cell_val2, lval.key, rval.s)
2046
2047	else:
2048	raise AssertionError(lval.tag())
2049
2050	def _BindNewArrayWithEntry(self, var_frame, lval, val, flags):
2051	# type: (Dict[str, Cell], sh_lvalue.Indexed, value.Str, int) -> None
2052	"""Fill 'var_frame' with a new indexed array entry."""
2053	no_str = None # type: Optional[str]
2054	items = [no_str] * lval.index
2055	items.append(val.s)
2056	new_value = value.BashArray(items)
2057
2058	# arrays can't be exported; can't have BashAssoc flag
2059	readonly = bool(flags & SetReadOnly)
2060	var_frame[lval.name] = Cell(False, readonly, False, new_value)
2061
2062	def InternalSetGlobal(self, name, new_val):
2063	# type: (str, value_t) -> None
2064	"""For setting read-only globals internally.
2065
2066	Args:
2067	name: string (not Lhs)
2068	new_val: value
2069
2070	The variable must already exist.
2071
2072	Use case: SHELLOPTS.
2073	"""
2074	cell = self.var_stack[0][name]
2075	cell.val = new_val
2076
2077	def GetValue(self, name, which_scopes=scope_e.Shopt):
2078	# type: (str, scope_t) -> value_t
2079	"""Used by the WordEvaluator, ArithEvaluator, ExprEvaluator, etc."""
2080	assert isinstance(name, str), name
2081
2082	if which_scopes == scope_e.Shopt:
2083	which_scopes = self.ScopesForReading()
2084
2085	with str_switch(name) as case:
2086	# "Registers"
2087	if case('_status'): # deprecated in favor of _error.code
2088	return num.ToBig(self.TryStatus())
2089
2090	elif case('_error'):
2091	return self.TryError()
2092
2093	elif case('_this_dir'):
2094	if len(self.this_dir) == 0:
2095	# e.g. osh -c '' doesn't have it set
2096	# Should we give a custom error here?
2097	# If you're at the interactive shell, 'source mymodule.ysh' will still
2098	# work because 'source' sets it.
2099	return value.Undef
2100	else:
2101	return value.Str(self.this_dir[-1]) # top of stack
2102
2103	elif case('PIPESTATUS'):
2104	strs2 = [str(i)
2105	for i in self.pipe_status[-1]] # type: List[str]
2106	return value.BashArray(strs2)
2107
2108	elif case('_pipeline_status'):
2109	items = [num.ToBig(i)
2110	for i in self.pipe_status[-1]] # type: List[value_t]
2111	return value.List(items)
2112
2113	elif case('_process_sub_status'): # YSH naming convention
2114	items = [num.ToBig(i) for i in self.process_sub_status[-1]]
2115	return value.List(items)
2116
2117	elif case('BASH_REMATCH'):
2118	top_match = self.regex_match[-1]
2119	with tagswitch(top_match) as case2:
2120	if case2(regex_match_e.No):
2121	groups = [] # type: List[str]
2122	elif case2(regex_match_e.Yes):
2123	m = cast(RegexMatch, top_match)
2124	groups = util.RegexGroupStrings(m.s, m.indices)
2125	return value.BashArray(groups)
2126
2127	# Do lookup of system globals before looking at user variables. Note: we
2128	# could optimize this at compile-time like $?. That would break
2129	# ${!varref}, but it's already broken for $?.
2130
2131	elif case('FUNCNAME'):
2132	# bash wants it in reverse order. This is a little inefficient but we're
2133	# not depending on deque().
2134	strs = [] # type: List[str]
2135	for frame in reversed(self.debug_stack):
2136	UP_frame = frame
2137	with tagswitch(frame) as case2:
2138	if case2(debug_frame_e.Call):
2139	frame = cast(debug_frame.Call, UP_frame)
2140	strs.append(frame.func_name)
2141
2142	elif case2(debug_frame_e.Source):
2143	# bash doesn't tell you the filename sourced
2144	strs.append('source')
2145
2146	elif case2(debug_frame_e.MainFile):
2147	strs.append('main') # also bash behavior
2148
2149	else: # ignore
2150	pass
2151
2152	return value.BashArray(strs) # TODO: Reuse this object too?
2153
2154	# $BASH_SOURCE and $BASH_LINENO have OFF BY ONE design bugs:
2155	#
2156	# ${BASH_LINENO[$i]} is the line number in the source file
2157	# (${BASH_SOURCE[$i+1]}) where ${FUNCNAME[$i]} was called (or
2158	# ${BASH_LINENO[$i-1]} if referenced within another shell function).
2159	#
2160	# https://www.gnu.org/software/bash/manual/html_node/Bash-Variables.html
2161
2162	elif case('BASH_SOURCE'):
2163	strs = []
2164	for frame in reversed(self.debug_stack):
2165	UP_frame = frame
2166	with tagswitch(frame) as case2:
2167	if case2(debug_frame_e.Call):
2168	frame = cast(debug_frame.Call, UP_frame)
2169
2170	# Weird bash behavior
2171	assert frame.def_tok.line is not None
2172	source_str = ui.GetLineSourceString(
2173	frame.def_tok.line)
2174	strs.append(source_str)
2175
2176	elif case2(debug_frame_e.Source):
2177	frame = cast(debug_frame.Source, UP_frame)
2178	# Is this right?
2179	strs.append(frame.source_name)
2180
2181	elif case2(debug_frame_e.MainFile):
2182	frame = cast(debug_frame.MainFile, UP_frame)
2183	strs.append(frame.main_filename)
2184
2185	else: # ignore
2186	pass
2187
2188	return value.BashArray(strs) # TODO: Reuse this object too?
2189
2190	elif case('BASH_LINENO'):
2191	strs = []
2192	for frame in reversed(self.debug_stack):
2193	UP_frame = frame
2194	with tagswitch(frame) as case2:
2195	if case2(debug_frame_e.Call):
2196	frame = cast(debug_frame.Call, UP_frame)
2197	strs.append(_LineNumber(frame.call_tok))
2198
2199	elif case2(debug_frame_e.Source):
2200	frame = cast(debug_frame.Source, UP_frame)
2201	strs.append(_LineNumber(frame.call_tok))
2202
2203	elif case2(debug_frame_e.MainFile):
2204	# Bash does this to line up with 'main'
2205	strs.append('0')
2206
2207	else: # ignore
2208	pass
2209
2210	return value.BashArray(strs) # TODO: Reuse this object too?
2211
2212	elif case('LINENO'):
2213	assert self.token_for_line is not None
2214	# Reuse object with mutation
2215	# TODO: maybe use interned GetLineNumStr?
2216	self.line_num.s = str(self.token_for_line.line.line_num)
2217	return self.line_num
2218
2219	elif case('BASHPID'): # TODO: YSH io->getpid()
2220	return value.Str(str(posix.getpid()))
2221
2222	elif case('_'):
2223	return value.Str(self.last_arg)
2224
2225	elif case('SECONDS'):
2226	f = time_.time() - self.seconds_start
2227	ok, big_int = mops.FromFloat(f)
2228	assert ok, f # should never be NAN or INFINITY
2229	return value.Int(big_int)
2230
2231	else:
2232	# In the case 'declare -n ref='a[42]', the result won't be a cell. Idea to
2233	# fix this:
2234	# 1. Call self.unsafe_arith.ParseVarRef() -> BracedVarSub
2235	# 2. Call self.unsafe_arith.GetNameref(bvs_part), and get a value_t
2236	# We still need a ref_trail to detect cycles.
2237	cell, _, _ = self._ResolveNameOrRef(name, which_scopes)
2238	if cell:
2239	return cell.val
2240
2241	builtin_val = self.builtins.get(name)
2242	if builtin_val:
2243	return builtin_val
2244
2245	# TODO: Can look in the builtins module, which is a value.Obj
2246	return value.Undef
2247
2248	def GetCell(self, name, which_scopes=scope_e.Shopt):
2249	# type: (str, scope_t) -> Cell
2250	"""Get both the value and flags.
2251
2252	Usages:
2253	- the 'pp' builtin.
2254	- declare -p
2255	- ${x@a}
2256	- to test of 'TZ' is exported in printf? Why?
2257
2258	Note: consulting __builtins__ doesn't see necessary for any of these
2259	"""
2260	if which_scopes == scope_e.Shopt:
2261	which_scopes = self.ScopesForReading()
2262
2263	cell, _ = self._ResolveNameOnly(name, which_scopes)
2264	return cell
2265
2266	def Unset(self, lval, which_scopes):
2267	# type: (sh_lvalue_t, scope_t) -> bool
2268	"""
2269	Returns:
2270	Whether the cell was found.
2271	"""
2272	# TODO: Refactor sh_lvalue type to avoid this
2273	UP_lval = lval
2274
2275	with tagswitch(lval) as case:
2276	if case(sh_lvalue_e.Var): # unset x
2277	lval = cast(LeftName, UP_lval)
2278	var_name = lval.name
2279	elif case(sh_lvalue_e.Indexed): # unset 'a[1]'
2280	lval = cast(sh_lvalue.Indexed, UP_lval)
2281	var_name = lval.name
2282	elif case(sh_lvalue_e.Keyed): # unset 'A["K"]'
2283	lval = cast(sh_lvalue.Keyed, UP_lval)
2284	var_name = lval.name
2285	else:
2286	raise AssertionError()
2287
2288	if which_scopes == scope_e.Shopt:
2289	which_scopes = self.ScopesForWriting()
2290
2291	cell, var_frame, cell_name = self._ResolveNameOrRef(
2292	var_name, which_scopes)
2293	if not cell:
2294	return False # 'unset' builtin falls back on functions
2295	if cell.readonly:
2296	raise error.Runtime("Can't unset readonly variable %r" % var_name)
2297
2298	with tagswitch(lval) as case:
2299	if case(sh_lvalue_e.Var): # unset x
2300	# Make variables in higher scopes visible.
2301	# example: test/spec.sh builtin-vars -r 24 (ble.sh)
2302	mylib.dict_erase(var_frame, cell_name)
2303
2304	# alternative that some shells use:
2305	# var_frame[cell_name].val = value.Undef
2306	# cell.exported = False
2307
2308	# This should never happen because we do recursive lookups of namerefs.
2309	assert not cell.nameref, cell
2310
2311	elif case(sh_lvalue_e.Indexed): # unset 'a[1]'
2312	lval = cast(sh_lvalue.Indexed, UP_lval)
2313	# Note: Setting an entry to None and shifting entries are pretty
2314	# much the same in shell.
2315
2316	val = cell.val
2317	UP_val = val
2318	if val.tag() == value_e.BashArray:
2319	val = cast(value.BashArray, UP_val)
2320	error_code = bash_impl.BashArray_UnsetElement(
2321	val, lval.index)
2322	if error_code == error_code_e.IndexOutOfRange:
2323	n = bash_impl.BashArray_Length(val)
2324	raise error.Runtime(
2325	"%s[%d]: Index is out of bounds for array of length %d"
2326	% (var_name, lval.index, n))
2327	elif val.tag() == value_e.SparseArray:
2328	val = cast(value.SparseArray, UP_val)
2329	error_code = bash_impl.SparseArray_UnsetElement(
2330	val, mops.IntWiden(lval.index))
2331	if error_code == error_code_e.IndexOutOfRange:
2332	big_length = bash_impl.SparseArray_Length(val)
2333	raise error.Runtime(
2334	"%s[%d]: Index is out of bounds for array of length %s"
2335	% (var_name, lval.index, mops.ToStr(big_length)))
2336	else:
2337	raise error.Runtime("%r isn't an array" % var_name)
2338
2339	elif case(sh_lvalue_e.Keyed): # unset 'A["K"]'
2340	lval = cast(sh_lvalue.Keyed, UP_lval)
2341
2342	val = cell.val
2343	UP_val = val
2344
2345	# note: never happens because of mem.IsBashAssoc test for sh_lvalue.Keyed
2346	#if val.tag() != value_e.BashAssoc:
2347	# raise error.Runtime("%r isn't an associative array" % lval.name)
2348
2349	val = cast(value.BashAssoc, UP_val)
2350	bash_impl.BashAssoc_UnsetElement(val, lval.key)
2351
2352	else:
2353	raise AssertionError(lval)
2354
2355	return True
2356
2357	def ScopesForReading(self):
2358	# type: () -> scope_t
2359	"""Read scope."""
2360	return (scope_e.Dynamic
2361	if self.exec_opts.dynamic_scope() else scope_e.LocalOrGlobal)
2362
2363	def ScopesForWriting(self):
2364	# type: () -> scope_t
2365	"""Write scope."""
2366	return (scope_e.Dynamic
2367	if self.exec_opts.dynamic_scope() else scope_e.LocalOnly)
2368
2369	def ClearFlag(self, name, flag):
2370	# type: (str, int) -> bool
2371	"""Used for export -n.
2372
2373	We don't use SetValue() because even if rval is None, it will make an
2374	Undef value in a scope.
2375	"""
2376	cell, var_frame = self._ResolveNameOnly(name, self.ScopesForReading())
2377	if cell:
2378	if flag & ClearExport:
2379	cell.exported = False
2380	if flag & ClearNameref:
2381	cell.nameref = False
2382	return True
2383	else:
2384	return False
2385
2386	def _FillWithExported(self, new_env):
2387	# type: (Dict[str, str]) -> None
2388
2389	# Search from globals up. Names higher on the stack will overwrite
2390	# names lower on the stack.
2391	for scope in self.var_stack:
2392	for name, cell in iteritems(scope):
2393	if cell.exported and cell.val.tag() == value_e.Str:
2394	val = cast(value.Str, cell.val)
2395	new_env[name] = val.s
2396
2397	def _FillEnvObj(self, new_env, env_object):
2398	# type: (Dict[str, str], Obj) -> None
2399
2400	# Do the LEAST visible parts first
2401	if env_object.prototype is not None:
2402	self._FillEnvObj(new_env, env_object.prototype)
2403
2404	# Overwrite with MOST visible parts
2405	for name, val in iteritems(env_object.d):
2406	if val.tag() != value_e.Str:
2407	continue
2408	new_env[name] = cast(value.Str, val).s
2409
2410	def GetEnv(self):
2411	# type: () -> Dict[str, str]
2412	"""
2413	Get the environment that should be used for launching processes.
2414
2415	Note: This is run on every SimpleCommand. Should we have a dirty
2416	flag? We could notice these things:
2417
2418	- If an exported variable is changed
2419	- If the set of exported variables changes.
2420	"""
2421	new_env = {} # type: Dict[str, str]
2422
2423	# Note: ysh:upgrade has both of these behaviors
2424
2425	# OSH: Consult exported vars
2426	if not self.exec_opts.no_exported():
2427	self._FillWithExported(new_env)
2428
2429	# YSH: Consult the ENV dict
2430	if self.exec_opts.env_obj():
2431	self._FillEnvObj(new_env, self.env_object)
2432
2433	return new_env
2434
2435	def VarNames(self):
2436	# type: () -> List[str]
2437	"""For internal OSH completion and compgen -A variable.
2438
2439	NOTE: We could also add $? $$ etc.?
2440	"""
2441	ret = [] # type: List[str]
2442	# Look up the stack, yielding all variables. Bash seems to do this.
2443	for scope in self.var_stack:
2444	for name in scope:
2445	ret.append(name)
2446	return ret
2447
2448	def VarNamesStartingWith(self, prefix):
2449	# type: (str) -> List[str]
2450	"""For ${!prefix@}"""
2451	# Look up the stack, yielding all variables. Bash seems to do this.
2452	names = [] # type: List[str]
2453	for scope in self.var_stack:
2454	for name in scope:
2455	if name.startswith(prefix):
2456	names.append(name)
2457	return names
2458
2459	def GetAllVars(self):
2460	# type: () -> Dict[str, str]
2461	"""Get all variables and their values, for 'set' builtin."""
2462	result = {} # type: Dict[str, str]
2463	for scope in self.var_stack:
2464	for name, cell in iteritems(scope):
2465	# TODO: Show other types?
2466	val = cell.val
2467	if val.tag() == value_e.Str:
2468	str_val = cast(value.Str, val)
2469	result[name] = str_val.s
2470	return result
2471
2472	def GetAllCells(self, which_scopes):
2473	# type: (scope_t) -> Dict[str, Cell]
2474	"""Get all variables and their values, for 'set' builtin."""
2475	result = {} # type: Dict[str, Cell]
2476
2477	if which_scopes == scope_e.Dynamic:
2478	scopes = self.var_stack
2479	elif which_scopes == scope_e.LocalOnly:
2480	scopes = self.var_stack[-1:]
2481	elif which_scopes == scope_e.GlobalOnly:
2482	scopes = self.var_stack[0:1]
2483	elif which_scopes == scope_e.LocalOrGlobal:
2484	scopes = [self.var_stack[0]]
2485	if len(self.var_stack) > 1:
2486	scopes.append(self.var_stack[-1])
2487	else:
2488	raise AssertionError()
2489
2490	for scope in scopes:
2491	for name, cell in iteritems(scope):
2492	result[name] = cell
2493	return result
2494
2495	def SetRegexMatch(self, match):
2496	# type: (regex_match_t) -> None
2497	self.regex_match[-1] = match
2498
2499	def GetRegexMatch(self):
2500	# type: () -> regex_match_t
2501	return self.regex_match[-1]
2502
2503	def PushContextStack(self, context):
2504	# type: (Dict[str, value_t]) -> None
2505	self.ctx_stack.append(context)
2506
2507	def GetContext(self):
2508	# type: () -> Optional[Dict[str, value_t]]
2509	if len(self.ctx_stack):
2510	return self.ctx_stack[-1]
2511	return None
2512
2513	def PopContextStack(self):
2514	# type: () -> Dict[str, value_t]
2515	assert self.ctx_stack, "Empty context stack"
2516	return self.ctx_stack.pop()
2517
2518
2519	def ValueIsInvokableObj(val):
2520	# type: (value_t) -> Tuple[Optional[value_t], Optional[Obj]]
2521	"""
2522	Returns:
2523	(__invoke__ Proc or BuiltinProc, self Obj) if the value is invokable
2524	(None, None) otherwise
2525	"""
2526	if val.tag() != value_e.Obj:
2527	return None, None
2528
2529	obj = cast(Obj, val)
2530	if not obj.prototype:
2531	return None, None
2532
2533	invoke_val = obj.prototype.d.get('__invoke__')
2534	if invoke_val is None:
2535	return None, None
2536
2537	# TODO: __invoke__ of wrong type could be fatal error?
2538	if invoke_val.tag() in (value_e.Proc, value_e.BuiltinProc):
2539	return invoke_val, obj
2540
2541	return None, None
2542
2543
2544	def _AddNames(unique, frame):
2545	# type: (Dict[str, bool], Dict[str, Cell]) -> None
2546	for name in frame:
2547	val = frame[name].val
2548	if val.tag() == value_e.Proc:
2549	unique[name] = True
2550	proc, _ = ValueIsInvokableObj(val)
2551	if proc is not None:
2552	unique[name] = True
2553
2554
2555	class Procs(object):
2556	"""
2557	Terminology:
2558
2559	- invokable - these are INTERIOR
2560	- value.Proc - which can be shell function in __sh_funcs__ namespace, or
2561	YSH proc
2562	- value.Obj with __invoke__
2563	- exterior - external commands, extern builtin
2564
2565	Note: the YSH 'invoke' builtin can generalize YSH 'runproc' builtin, shell command/builtin,
2566	and also type / type -a
2567	"""
2568
2569	def __init__(self, mem):
2570	# type: (Mem) -> None
2571	self.mem = mem
2572	self.sh_funcs = {} # type: Dict[str, value.Proc]
2573
2574	def DefineShellFunc(self, name, proc):
2575	# type: (str, value.Proc) -> None
2576	self.sh_funcs[name] = proc
2577
2578	def IsShellFunc(self, name):
2579	# type: (str) -> bool
2580	return name in self.sh_funcs
2581
2582	def GetShellFunc(self, name):
2583	# type: (str) -> Optional[value.Proc]
2584	return self.sh_funcs.get(name)
2585
2586	def EraseShellFunc(self, to_del):
2587	# type: (str) -> None
2588	"""Undefine a sh-func with name `to_del`, if it exists."""
2589	mylib.dict_erase(self.sh_funcs, to_del)
2590
2591	def ShellFuncNames(self):
2592	# type: () -> List[str]
2593	"""Returns a sorted list of all shell function names
2594
2595	Callers:
2596	declare -f -F
2597	"""
2598	names = self.sh_funcs.keys()
2599	names.sort()
2600	return names
2601
2602	def DefineProc(self, name, proc):
2603	# type: (str, value.Proc) -> None
2604	"""
2605	procs are defined in the local scope.
2606	"""
2607	self.mem.var_stack[-1][name] = Cell(False, False, False, proc)
2608
2609	def IsProc(self, name):
2610	# type: (str) -> bool
2611
2612	maybe_proc = self.mem.GetValue(name)
2613	# Could be Undef
2614	return maybe_proc.tag() == value_e.Proc
2615
2616	def IsInvokableObj(self, name):
2617	# type: (str) -> bool
2618
2619	val = self.mem.GetValue(name)
2620	proc, _ = ValueIsInvokableObj(val)
2621	return proc is not None
2622
2623	def InvokableNames(self):
2624	# type: () -> List[str]
2625	"""Returns a sorted list of all invokable names
2626
2627	Callers:
2628	complete -A function
2629	pp proc - should deprecate this
2630	"""
2631	unique = NewDict() # type: Dict[str, bool]
2632	for name in self.sh_funcs:
2633	unique[name] = True
2634
2635	top_frame = self.mem.var_stack[-1]
2636	_AddNames(unique, top_frame)
2637
2638	global_frame = self.mem.var_stack[0]
2639	#log('%d %d', id(top_frame), id(global_frame))
2640	if global_frame is not top_frame:
2641	_AddNames(unique, global_frame)
2642
2643	#log('%s', unique)
2644
2645	names = unique.keys()
2646	names.sort()
2647
2648	return names
2649
2650	def GetInvokable(self, name):
2651	# type: (str) -> Tuple[Optional[value_t], Optional[Obj]]
2652	"""Find a proc, invokable Obj, or sh-func, in that order
2653
2654	Callers:
2655	executor.py: to actually run
2656	meta_oils.py runproc lookup - this is not 'invoke', because it is
2657	INTERIOR shell functions, procs, invokable Obj
2658	"""
2659	val = self.mem.GetValue(name)
2660
2661	if val.tag() == value_e.Proc:
2662	return cast(value.Proc, val), None
2663
2664	proc, self_val = ValueIsInvokableObj(val)
2665	if proc:
2666	return proc, self_val
2667
2668	if name in self.sh_funcs:
2669	return self.sh_funcs[name], None
2670
2671	return None, None
2672
2673
2674	#
2675	# Wrappers to Set Variables
2676	#
2677
2678
2679	def OshLanguageSetValue(mem, lval, val, flags=0):
2680	# type: (Mem, sh_lvalue_t, value_t, int) -> None
2681	"""Like 'setvar' (scope_e.LocalOnly), unless dynamic scope is on.
2682
2683	That is, it respects shopt --unset dynamic_scope.
2684
2685	Used for assignment builtins, (( a = b )), {fd}>out, ${x=}, etc.
2686	"""
2687	which_scopes = mem.ScopesForWriting()
2688	mem.SetValue(lval, val, which_scopes, flags=flags)
2689
2690
2691	def BuiltinSetValue(mem, lval, val):
2692	# type: (Mem, sh_lvalue_t, value_t) -> None
2693	"""Equivalent of x=$y
2694
2695	Called by BuiltinSetString and BuiltinSetArray Used directly by
2696	printf -v because it can mutate an array
2697	"""
2698	mem.SetValue(lval, val, mem.ScopesForWriting())
2699
2700
2701	def BuiltinSetString(mem, name, s):
2702	# type: (Mem, str, str) -> None
2703	"""Set a string by looking up the stack.
2704
2705	Used for 'read', 'getopts', completion builtins, etc.
2706	"""
2707	assert isinstance(s, str)
2708	BuiltinSetValue(mem, location.LName(name), value.Str(s))
2709
2710
2711	def BuiltinSetArray(mem, name, a):
2712	# type: (Mem, str, List[str]) -> None
2713	"""Set an array by looking up the stack.
2714
2715	Used by compadjust, read -a, etc.
2716	"""
2717	assert isinstance(a, list)
2718	BuiltinSetValue(mem, location.LName(name), value.BashArray(a))
2719
2720
2721	def SetGlobalString(mem, name, s):
2722	# type: (Mem, str, str) -> None
2723	"""Helper for completion, etc."""
2724	assert isinstance(s, str)
2725	val = value.Str(s)
2726	mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly)
2727
2728
2729	def SetGlobalArray(mem, name, a):
2730	# type: (Mem, str, List[str]) -> None
2731	"""Used by completion, shell initialization, etc."""
2732	assert isinstance(a, list)
2733	mem.SetNamed(location.LName(name), value.BashArray(a), scope_e.GlobalOnly)
2734
2735
2736	def SetGlobalValue(mem, name, val):
2737	# type: (Mem, str, value_t) -> None
2738	"""Helper for completion, etc."""
2739	mem.SetNamed(location.LName(name), val, scope_e.GlobalOnly)
2740
2741
2742	def SetLocalValue(mem, name, val):
2743	# type: (Mem, str, value_t) -> None
2744	"""For 'use' builtin."""
2745	mem.SetNamed(location.LName(name), val, scope_e.LocalOnly)
2746
2747
2748	def ExportGlobalString(mem, name, s):
2749	# type: (Mem, str, str) -> None
2750	"""Helper for completion, $PWD, $OLDPWD, etc."""
2751	assert isinstance(s, str)
2752	val = value.Str(s)
2753	mem.SetNamed(location.LName(name),
2754	val,
2755	scope_e.GlobalOnly,
2756	flags=SetExport)
2757
2758
2759	# TODO: remove in favor of EnvConfig
2760	def SetStringInEnv(mem, var_name, s):
2761	# type: (Mem, str, str) -> None
2762	if mem.exec_opts.env_obj(): # e.g. ENV.YSH_HISTFILE
2763	mem.env_dict[var_name] = value.Str(s)
2764	else: # e.g. $YSH_HISTFILE
2765	SetGlobalString(mem, var_name, s)
2766
2767
2768	#
2769	# Wrappers to Get Variables
2770	#
2771
2772
2773	def DynamicGetVar(mem, name, which_scopes):
2774	# type: (Mem, str, scope_t) -> value_t
2775	"""
2776	For getVar() and shvarGet()
2777	"""
2778	val = mem.GetValue(name, which_scopes=which_scopes)
2779
2780	# Undef is not a user-visible value!
2781	# There's no way to distinguish null from undefined.
2782	if val.tag() == value_e.Undef:
2783	return value.Null
2784
2785	return val
2786
2787
2788	def GetString(mem, name):
2789	# type: (Mem, str) -> str
2790	"""Wrapper around GetValue().
2791
2792	Check that HOME, PWD, OLDPWD, etc. are strings. bash doesn't have these
2793	errors because ${array} is ${array[0]}.
2794
2795	TODO: We could also check this when you're storing variables?
2796	"""
2797	val = mem.GetValue(name)
2798	UP_val = val
2799	with tagswitch(val) as case:
2800	if case(value_e.Undef):
2801	raise error.Runtime("$%s isn't defined" % name)
2802	elif case(value_e.Str):
2803	return cast(value.Str, UP_val).s
2804	else:
2805	# User would have to 'unset HOME' to get rid of exported flag
2806	raise error.Runtime("$%s should be a string" % name)
2807
2808
2809	def MaybeString(mem, name):
2810	# type: (Mem, str) -> Optional[str]
2811	"""Like GetString(), but doesn't throw an exception."""
2812	try:
2813	return GetString(mem, name)
2814	except error.Runtime:
2815	return None
2816
2817
2818	def GetInteger(mem, name):
2819	# type: (Mem, str) -> int
2820	"""For OPTIND variable used in getopts builtin.
2821
2822	TODO: it could be value.Int() ?
2823	"""
2824	val = mem.GetValue(name)
2825	if val.tag() != value_e.Str:
2826	raise error.Runtime('$%s should be a string, got %s' %
2827	(name, ui.ValType(val)))
2828	s = cast(value.Str, val).s
2829	try:
2830	i = int(s)
2831	except ValueError:
2832	raise error.Runtime("$%s doesn't look like an integer, got %r" %
2833	(name, s))
2834	return i
2835
2836
2837	# vim: sw=4