OILS / core / test_lib.py View on Github | oilshell.org

402 lines, 280 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value, Obj
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyutil
35from core import sh_init
36from core import state
37from display import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import iolib
51from mycpp import mylib
52
53import posix_ as posix
54
55
56def MakeBuiltinArgv(argv):
57 return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None)
58
59
60def FakeTok(id_, val):
61 # type: (int, str) -> Token
62 src = source.Interactive
63 line = SourceLine(1, val, src)
64 return Token(id_, len(val), 0, line, None)
65
66
67def PrintableString(s):
68 """For pretty-printing in tests."""
69 if all(c in string.printable for c in s):
70 return s
71 return repr(s)
72
73
74def TokensEqual(left, right):
75 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
76
77 if left.id != right.id:
78 return False
79
80 if left.line is not None:
81 left_str = lexer.TokenVal(left)
82 else:
83 left_str = None
84
85 if right.line is not None:
86 right_str = lexer.TokenVal(right)
87 else:
88 right_str = None
89
90 # Better error message sometimes:
91 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
92 return left_str == right_str
93
94
95def TokenWordsEqual(left, right):
96 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
97 return TokensEqual(left.token, right.token)
98 #return left == right
99
100
101def AsdlEqual(left, right):
102 """Check if generated ASDL instances are equal.
103
104 We don't use equality in the actual code, so this is relegated to
105 test_lib.
106 """
107 if left is None and right is None:
108 return True
109
110 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
111 return left == right
112
113 if isinstance(left, list):
114 if len(left) != len(right):
115 return False
116 for a, b in zip(left, right):
117 if not AsdlEqual(a, b):
118 return False
119 return True
120
121 if isinstance(left, pybase.CompoundObj):
122 if left.tag() != right.tag():
123 return False
124
125 field_names = left.__slots__ # hack for now
126 for name in field_names:
127 # Special case: we are not testing locations right now.
128 if name == 'span_id':
129 continue
130 a = getattr(left, name)
131 b = getattr(right, name)
132 if not AsdlEqual(a, b):
133 return False
134
135 return True
136
137 raise AssertionError(left)
138
139
140def AssertAsdlEqual(test, left, right):
141 test.assertTrue(AsdlEqual(left, right),
142 'Expected %s, got %s' % (left, right))
143
144
145def MakeArena(source_name):
146 arena = alloc.Arena(save_tokens=True)
147 arena.PushSource(source.MainFile(source_name))
148 return arena
149
150
151def InitLineLexer(s, arena):
152 line_lexer = lexer.LineLexer(arena)
153 src = source.Interactive
154 line_lexer.Reset(SourceLine(1, s, src), 0)
155 return line_lexer
156
157
158def InitLexer(s, arena):
159 """For tests only."""
160 line_lexer = lexer.LineLexer(arena)
161 line_reader = reader.StringLineReader(s, arena)
162 lx = lexer.Lexer(line_lexer, line_reader)
163 return line_reader, lx
164
165
166def InitWordEvaluator(exec_opts=None):
167 arena = MakeArena('<InitWordEvaluator>')
168 mem = state.Mem('', [], arena, [], {})
169
170 if exec_opts is None:
171 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
172 mem.exec_opts = exec_opts # circular dep
173 sh_init.InitDefaultVars(mem)
174 mutable_opts.Init()
175 else:
176 mutable_opts = None
177
178 cmd_deps = cmd_eval.Deps()
179 cmd_deps.trap_nodes = []
180
181 splitter = split.SplitContext(mem)
182 errfmt = ui.ErrorFormatter()
183
184 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
185 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
186 tilde_ev, splitter, errfmt)
187 return ev
188
189
190def InitCommandEvaluator(parse_ctx=None,
191 comp_lookup=None,
192 arena=None,
193 mem=None,
194 aliases=None,
195 ext_prog=None):
196
197 opt0_array = state.InitOpts()
198 opt_stacks = [None] * option_i.ARRAY_SIZE
199 if parse_ctx:
200 arena = parse_ctx.arena
201 else:
202 parse_ctx = InitParseContext()
203
204 mem = mem or state.Mem('', [], arena, [], {})
205 exec_opts = optview.Exec(opt0_array, opt_stacks)
206 mutable_opts = state.MutableOpts(mem, {}, opt0_array, opt_stacks, None)
207 mem.exec_opts = exec_opts
208 #state.InitMem(mem, {}, '0.1')
209 sh_init.InitDefaultVars(mem)
210 mutable_opts.Init()
211
212 # No 'readline' in the tests.
213
214 errfmt = ui.ErrorFormatter()
215 job_control = process.JobControl()
216 job_list = process.JobList()
217 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
218 exec_opts)
219 aliases = {} if aliases is None else aliases
220 procs = state.Procs(mem)
221 methods = {}
222
223 compopt_state = completion.OptionState()
224 comp_lookup = comp_lookup or completion.Lookup()
225
226 readline = None # simulate not having it
227
228 new_var = assign_osh.NewVar(mem, procs, exec_opts, errfmt)
229 assign_builtins = {
230 builtin_i.declare: new_var,
231 builtin_i.typeset: new_var,
232 builtin_i.local: new_var,
233 builtin_i.export_: assign_osh.Export(mem, errfmt),
234 builtin_i.readonly: assign_osh.Readonly(mem, errfmt),
235 }
236 builtins = { # Lookup
237 builtin_i.echo: io_osh.Echo(exec_opts),
238 builtin_i.shift: assign_osh.Shift(mem),
239
240 builtin_i.history: readline_osh.History(
241 readline,
242 mem,
243 errfmt,
244 mylib.Stdout(),
245 ),
246
247 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
248 builtin_i.compadjust: completion_osh.CompAdjust(mem),
249
250 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
251 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
252 }
253
254 debug_f = util.DebugFile(sys.stderr)
255 cmd_deps = cmd_eval.Deps()
256 cmd_deps.mutable_opts = mutable_opts
257
258 search_path = executor.SearchPath(mem, exec_opts)
259
260 ext_prog = \
261 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
262
263 cmd_deps.dumper = dev.CrashDumper('', fd_state)
264 cmd_deps.debug_f = debug_f
265
266 splitter = split.SplitContext(mem)
267
268 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
269 parse_ctx, errfmt)
270 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
271 parse_ctx, errfmt)
272 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
273 errfmt)
274 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
275 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
276 tilde_ev, splitter, errfmt)
277 signal_safe = iolib.InitSignalSafe()
278 trap_state = trap_osh.TrapState(signal_safe)
279 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
280 assign_builtins, arena, cmd_deps,
281 trap_state, signal_safe)
282
283 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
284 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
285 multi_trace)
286 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
287
288 hay_state = hay_ysh.HayState()
289 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
290 hay_state, builtins, search_path,
291 ext_prog, waiter, tracer, job_control,
292 job_list, fd_state, trap_state, errfmt)
293
294 assert cmd_ev.mutable_opts is not None, cmd_ev
295 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
296
297 global_io = Obj(None, None)
298 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
299 prompt_ev, global_io, tracer)
300
301 try:
302 from _devbuild.gen.help_meta import TOPICS
303 except ImportError:
304 TOPICS = None # minimal dev build
305 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
306 splitter, comp_lookup, TOPICS,
307 errfmt)
308
309 # Add some builtins that depend on the executor!
310 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
311 builtins[builtin_i.complete] = complete_builtin
312 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
313
314 return cmd_ev
315
316
317def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
318 """Unit tests can evaluate code strings and then use the resulting
319 CommandEvaluator."""
320 arena = parse_ctx.arena
321 errfmt = ui.ErrorFormatter()
322
323 comp_lookup = comp_lookup or completion.Lookup()
324 mem = mem or state.Mem('', [], arena, [], {})
325 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
326 mem.exec_opts = exec_opts
327
328 #state.InitMem(mem, {}, '0.1')
329 sh_init.InitDefaultVars(mem)
330 mutable_opts.Init()
331
332 line_reader, _ = InitLexer(code_str, arena)
333 c_parser = parse_ctx.MakeOshParser(line_reader)
334
335 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
336 comp_lookup=comp_lookup,
337 arena=arena,
338 mem=mem,
339 aliases=aliases)
340
341 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
342 return cmd_ev
343
344
345def InitParseContext(arena=None,
346 ysh_grammar=None,
347 aliases=None,
348 parse_opts=None,
349 do_lossless=False):
350 arena = arena or MakeArena('<test_lib>')
351
352 if aliases is None:
353 aliases = {}
354
355 mem = state.Mem('', [], arena, [], {})
356 if parse_opts is None:
357 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
358
359 parse_ctx = parse_lib.ParseContext(arena,
360 parse_opts,
361 aliases,
362 ysh_grammar,
363 do_lossless=do_lossless)
364
365 return parse_ctx
366
367
368def InitWordParser(word_str, oil_at=False, arena=None):
369 arena = arena or MakeArena('<test_lib>')
370
371 mem = state.Mem('', [], arena, [], {})
372 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
373
374 # CUSTOM SETTING
375 mutable_opts.opt0_array[option_i.parse_at] = oil_at
376
377 loader = pyutil.GetResourceLoader()
378 ysh_grammar = pyutil.LoadYshGrammar(loader)
379 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
380 line_reader, _ = InitLexer(word_str, arena)
381 c_parser = parse_ctx.MakeOshParser(line_reader)
382 # Hack
383 return c_parser.w_parser
384
385
386def InitCommandParser(code_str, arena=None):
387 arena = arena or MakeArena('<test_lib>')
388
389 loader = pyutil.GetResourceLoader()
390 ysh_grammar = pyutil.LoadYshGrammar(loader)
391
392 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
393 line_reader, _ = InitLexer(code_str, arena)
394 c_parser = parse_ctx.MakeOshParser(line_reader)
395 return c_parser
396
397
398def SetLocalString(mem, name, s):
399 # type: (state.Mem, str, str) -> None
400 """Bind a local string."""
401 assert isinstance(s, str)
402 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)