OILS / core / test_lib.py View on Github | oilshell.org

401 lines, 279 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value, Obj
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyos
35from core import pyutil
36from core import state
37from display import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import mylib
51
52import posix_ as posix
53
54
55def MakeBuiltinArgv(argv):
56 return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None)
57
58
59def FakeTok(id_, val):
60 # type: (int, str) -> Token
61 src = source.Interactive
62 line = SourceLine(1, val, src)
63 return Token(id_, len(val), 0, line, None)
64
65
66def PrintableString(s):
67 """For pretty-printing in tests."""
68 if all(c in string.printable for c in s):
69 return s
70 return repr(s)
71
72
73def TokensEqual(left, right):
74 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
75
76 if left.id != right.id:
77 return False
78
79 if left.line is not None:
80 left_str = lexer.TokenVal(left)
81 else:
82 left_str = None
83
84 if right.line is not None:
85 right_str = lexer.TokenVal(right)
86 else:
87 right_str = None
88
89 # Better error message sometimes:
90 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
91 return left_str == right_str
92
93
94def TokenWordsEqual(left, right):
95 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
96 return TokensEqual(left.token, right.token)
97 #return left == right
98
99
100def AsdlEqual(left, right):
101 """Check if generated ASDL instances are equal.
102
103 We don't use equality in the actual code, so this is relegated to
104 test_lib.
105 """
106 if left is None and right is None:
107 return True
108
109 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
110 return left == right
111
112 if isinstance(left, list):
113 if len(left) != len(right):
114 return False
115 for a, b in zip(left, right):
116 if not AsdlEqual(a, b):
117 return False
118 return True
119
120 if isinstance(left, pybase.CompoundObj):
121 if left.tag() != right.tag():
122 return False
123
124 field_names = left.__slots__ # hack for now
125 for name in field_names:
126 # Special case: we are not testing locations right now.
127 if name == 'span_id':
128 continue
129 a = getattr(left, name)
130 b = getattr(right, name)
131 if not AsdlEqual(a, b):
132 return False
133
134 return True
135
136 raise AssertionError(left)
137
138
139def AssertAsdlEqual(test, left, right):
140 test.assertTrue(AsdlEqual(left, right),
141 'Expected %s, got %s' % (left, right))
142
143
144def MakeArena(source_name):
145 arena = alloc.Arena(save_tokens=True)
146 arena.PushSource(source.MainFile(source_name))
147 return arena
148
149
150def InitLineLexer(s, arena):
151 line_lexer = lexer.LineLexer(arena)
152 src = source.Interactive
153 line_lexer.Reset(SourceLine(1, s, src), 0)
154 return line_lexer
155
156
157def InitLexer(s, arena):
158 """For tests only."""
159 line_lexer = lexer.LineLexer(arena)
160 line_reader = reader.StringLineReader(s, arena)
161 lx = lexer.Lexer(line_lexer, line_reader)
162 return line_reader, lx
163
164
165def InitWordEvaluator(exec_opts=None):
166 arena = MakeArena('<InitWordEvaluator>')
167 mem = state.Mem('', [], arena, [])
168
169 if exec_opts is None:
170 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
171 mem.exec_opts = exec_opts # circular dep
172 state.InitDefaultVars(mem)
173 mutable_opts.Init()
174 else:
175 mutable_opts = None
176
177 cmd_deps = cmd_eval.Deps()
178 cmd_deps.trap_nodes = []
179
180 splitter = split.SplitContext(mem)
181 errfmt = ui.ErrorFormatter()
182
183 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
184 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
185 tilde_ev, splitter, errfmt)
186 return ev
187
188
189def InitCommandEvaluator(parse_ctx=None,
190 comp_lookup=None,
191 arena=None,
192 mem=None,
193 aliases=None,
194 ext_prog=None):
195
196 opt0_array = state.InitOpts()
197 opt_stacks = [None] * option_i.ARRAY_SIZE
198 if parse_ctx:
199 arena = parse_ctx.arena
200 else:
201 parse_ctx = InitParseContext()
202
203 mem = mem or state.Mem('', [], arena, [])
204 exec_opts = optview.Exec(opt0_array, opt_stacks)
205 mutable_opts = state.MutableOpts(mem, opt0_array, opt_stacks, None)
206 mem.exec_opts = exec_opts
207 #state.InitMem(mem, {}, '0.1')
208 state.InitDefaultVars(mem)
209 mutable_opts.Init()
210
211 # No 'readline' in the tests.
212
213 errfmt = ui.ErrorFormatter()
214 job_control = process.JobControl()
215 job_list = process.JobList()
216 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
217 exec_opts)
218 aliases = {} if aliases is None else aliases
219 procs = state.Procs(mem)
220 methods = {}
221
222 compopt_state = completion.OptionState()
223 comp_lookup = comp_lookup or completion.Lookup()
224
225 readline = None # simulate not having it
226
227 new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
228 assign_builtins = {
229 builtin_i.declare: new_var,
230 builtin_i.typeset: new_var,
231 builtin_i.local: new_var,
232 builtin_i.export_: assign_osh.Export(mem, errfmt),
233 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
234 }
235 builtins = { # Lookup
236 builtin_i.echo: io_osh.Echo(exec_opts),
237 builtin_i.shift: assign_osh.Shift(mem),
238
239 builtin_i.history: readline_osh.History(
240 readline,
241 mem,
242 errfmt,
243 mylib.Stdout(),
244 ),
245
246 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
247 builtin_i.compadjust: completion_osh.CompAdjust(mem),
248
249 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
250 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
251 }
252
253 debug_f = util.DebugFile(sys.stderr)
254 cmd_deps = cmd_eval.Deps()
255 cmd_deps.mutable_opts = mutable_opts
256
257 search_path = state.SearchPath(mem)
258
259 ext_prog = \
260 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
261
262 cmd_deps.dumper = dev.CrashDumper('', fd_state)
263 cmd_deps.debug_f = debug_f
264
265 splitter = split.SplitContext(mem)
266
267 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
268 parse_ctx, errfmt)
269 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
270 parse_ctx, errfmt)
271 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
272 errfmt)
273 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
274 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
275 tilde_ev, splitter, errfmt)
276 signal_safe = pyos.InitSignalSafe()
277 trap_state = trap_osh.TrapState(signal_safe)
278 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
279 assign_builtins, arena, cmd_deps,
280 trap_state, signal_safe)
281
282 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
283 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
284 multi_trace)
285 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
286
287 hay_state = hay_ysh.HayState()
288 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
289 hay_state, builtins, search_path,
290 ext_prog, waiter, tracer, job_control,
291 job_list, fd_state, trap_state, errfmt)
292
293 assert cmd_ev.mutable_opts is not None, cmd_ev
294 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
295
296 global_io = Obj(None, None)
297 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
298 prompt_ev, global_io, tracer)
299
300 try:
301 from _devbuild.gen.help_meta import TOPICS
302 except ImportError:
303 TOPICS = None # minimal dev build
304 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
305 splitter, comp_lookup, TOPICS,
306 errfmt)
307
308 # Add some builtins that depend on the executor!
309 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
310 builtins[builtin_i.complete] = complete_builtin
311 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
312
313 return cmd_ev
314
315
316def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
317 """Unit tests can evaluate code strings and then use the resulting
318 CommandEvaluator."""
319 arena = parse_ctx.arena
320 errfmt = ui.ErrorFormatter()
321
322 comp_lookup = comp_lookup or completion.Lookup()
323 mem = mem or state.Mem('', [], arena, [])
324 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
325 mem.exec_opts = exec_opts
326
327 #state.InitMem(mem, {}, '0.1')
328 state.InitDefaultVars(mem)
329 mutable_opts.Init()
330
331 line_reader, _ = InitLexer(code_str, arena)
332 c_parser = parse_ctx.MakeOshParser(line_reader)
333
334 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
335 comp_lookup=comp_lookup,
336 arena=arena,
337 mem=mem,
338 aliases=aliases)
339
340 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
341 return cmd_ev
342
343
344def InitParseContext(arena=None,
345 ysh_grammar=None,
346 aliases=None,
347 parse_opts=None,
348 do_lossless=False):
349 arena = arena or MakeArena('<test_lib>')
350
351 if aliases is None:
352 aliases = {}
353
354 mem = state.Mem('', [], arena, [])
355 if parse_opts is None:
356 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
357
358 parse_ctx = parse_lib.ParseContext(arena,
359 parse_opts,
360 aliases,
361 ysh_grammar,
362 do_lossless=do_lossless)
363
364 return parse_ctx
365
366
367def InitWordParser(word_str, oil_at=False, arena=None):
368 arena = arena or MakeArena('<test_lib>')
369
370 mem = state.Mem('', [], arena, [])
371 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, None)
372
373 # CUSTOM SETTING
374 mutable_opts.opt0_array[option_i.parse_at] = oil_at
375
376 loader = pyutil.GetResourceLoader()
377 ysh_grammar = pyutil.LoadYshGrammar(loader)
378 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
379 line_reader, _ = InitLexer(word_str, arena)
380 c_parser = parse_ctx.MakeOshParser(line_reader)
381 # Hack
382 return c_parser.w_parser
383
384
385def InitCommandParser(code_str, arena=None):
386 arena = arena or MakeArena('<test_lib>')
387
388 loader = pyutil.GetResourceLoader()
389 ysh_grammar = pyutil.LoadYshGrammar(loader)
390
391 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
392 line_reader, _ = InitLexer(code_str, arena)
393 c_parser = parse_ctx.MakeOshParser(line_reader)
394 return c_parser
395
396
397def SetLocalString(mem, name, s):
398 # type: (state.Mem, str, str) -> None
399 """Bind a local string."""
400 assert isinstance(s, str)
401 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)