OILS / core / test_lib.py View on Github | oils.pub

413 lines, 287 significant
1#!/usr/bin/env python2
2# Copyright 2016 Andy Chu. All rights reserved.
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8"""
9test_lib.py - Functions for testing.
10"""
11
12import string
13import sys
14
15from _devbuild.gen.option_asdl import builtin_i, option_i
16from _devbuild.gen.runtime_asdl import cmd_value, scope_e
17from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
18from _devbuild.gen.value_asdl import value, Obj
19from asdl import pybase
20from builtin import assign_osh
21from builtin import completion_osh
22from builtin import hay_ysh
23from builtin import io_osh
24from builtin import pure_osh
25from builtin import readline_osh
26from builtin import trap_osh
27from core import alloc
28from core import completion
29from core import dev
30from core import executor
31from core import main_loop
32from core import optview
33from core import process
34from core import pyutil
35from core import sh_init
36from core import state
37from display import ui
38from core import util
39from core import vm
40from frontend import lexer
41from frontend import location
42from frontend import parse_lib
43from frontend import reader
44from osh import cmd_eval
45from osh import prompt
46from osh import sh_expr_eval
47from osh import split
48from osh import word_eval
49from ysh import expr_eval
50from mycpp import iolib
51from mycpp import mylib
52
53import posix_ as posix
54
55
56def MakeMem(arena):
57 return state.Mem('', [], arena, [], {})
58
59
60def MakeBuiltinArgv(argv):
61 return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None)
62
63
64def FakeTok(id_, val):
65 # type: (int, str) -> Token
66 src = source.Interactive
67 line = SourceLine(1, val, src)
68 return Token(id_, len(val), 0, line, None)
69
70
71def PrintableString(s):
72 """For pretty-printing in tests."""
73 if all(c in string.printable for c in s):
74 return s
75 return repr(s)
76
77
78def TokensEqual(left, right):
79 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
80
81 if left.id != right.id:
82 return False
83
84 if left.line is not None:
85 left_str = lexer.TokenVal(left)
86 else:
87 left_str = None
88
89 if right.line is not None:
90 right_str = lexer.TokenVal(right)
91 else:
92 right_str = None
93
94 # Better error message sometimes:
95 #assert left_str == right_str, '%r != %r' % (left_str, right_str)
96 return left_str == right_str
97
98
99def TokenWordsEqual(left, right):
100 # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
101 return TokensEqual(left.token, right.token)
102 #return left == right
103
104
105def AsdlEqual(left, right):
106 """Check if generated ASDL instances are equal.
107
108 We don't use equality in the actual code, so this is relegated to
109 test_lib.
110 """
111 if left is None and right is None:
112 return True
113
114 if isinstance(left, (int, str, bool, pybase.SimpleObj)):
115 return left == right
116
117 if isinstance(left, list):
118 if len(left) != len(right):
119 return False
120 for a, b in zip(left, right):
121 if not AsdlEqual(a, b):
122 return False
123 return True
124
125 if isinstance(left, pybase.CompoundObj):
126 if left.tag() != right.tag():
127 return False
128
129 field_names = left.__slots__ # hack for now
130 for name in field_names:
131 # Special case: we are not testing locations right now.
132 if name == 'span_id':
133 continue
134 a = getattr(left, name)
135 b = getattr(right, name)
136 if not AsdlEqual(a, b):
137 return False
138
139 return True
140
141 raise AssertionError(left)
142
143
144def AssertAsdlEqual(test, left, right):
145 test.assertTrue(AsdlEqual(left, right),
146 'Expected %s, got %s' % (left, right))
147
148
149def MakeArena(source_name):
150 arena = alloc.Arena(save_tokens=True)
151 arena.PushSource(source.MainFile(source_name))
152 return arena
153
154
155def InitLineLexer(s, arena):
156 line_lexer = lexer.LineLexer(arena)
157 src = source.Interactive
158 line_lexer.Reset(SourceLine(1, s, src), 0)
159 return line_lexer
160
161
162def InitLexer(s, arena):
163 """For tests only."""
164 line_lexer = lexer.LineLexer(arena)
165 line_reader = reader.StringLineReader(s, arena)
166 lx = lexer.Lexer(line_lexer, line_reader)
167 return line_reader, lx
168
169
170def InitWordEvaluator(exec_opts=None):
171 arena = MakeArena('<InitWordEvaluator>')
172 mem = MakeMem(arena)
173
174 if exec_opts is None:
175 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
176 mem.exec_opts = exec_opts # circular dep
177 sh_init.InitDefaultVars(mem, [])
178 mutable_opts.InitFromEnv('')
179 else:
180 mutable_opts = None
181
182 cmd_deps = cmd_eval.Deps()
183 cmd_deps.trap_nodes = []
184
185 splitter = split.SplitContext(mem)
186 errfmt = ui.ErrorFormatter()
187
188 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
189 ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
190 tilde_ev, splitter, errfmt)
191 return ev
192
193
194def InitCommandEvaluator(parse_ctx=None,
195 comp_lookup=None,
196 arena=None,
197 mem=None,
198 aliases=None,
199 ext_prog=None):
200
201 opt0_array = state.InitOpts()
202 opt_stacks = [None] * option_i.ARRAY_SIZE
203 if parse_ctx:
204 arena = parse_ctx.arena
205 else:
206 parse_ctx = InitParseContext()
207
208 mem = mem or MakeMem(arena)
209 exec_opts = optview.Exec(opt0_array, opt_stacks)
210 mutable_opts = state.MutableOpts(mem, {}, opt0_array, opt_stacks, None)
211 mem.exec_opts = exec_opts
212 #state.InitMem(mem, {}, '0.1')
213 sh_init.InitDefaultVars(mem, [])
214 mutable_opts.InitFromEnv('')
215
216 # No 'readline' in the tests.
217
218 errfmt = ui.ErrorFormatter()
219 job_control = process.JobControl()
220 job_list = process.JobList()
221 fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
222 exec_opts)
223 aliases = {} if aliases is None else aliases
224 procs = state.Procs(mem)
225 methods = {}
226
227 compopt_state = completion.OptionState()
228 comp_lookup = comp_lookup or completion.Lookup()
229
230 readline = None # simulate not having it
231
232 arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
233 parse_ctx, errfmt)
234 new_var = assign_osh.NewVar(mem, procs, exec_opts, arith_ev, errfmt)
235 assign_builtins = {
236 builtin_i.declare: new_var,
237 builtin_i.typeset: new_var,
238 builtin_i.local: new_var,
239 builtin_i.export_: assign_osh.Export(mem, arith_ev, errfmt),
240 builtin_i.readonly: assign_osh.Readonly(mem, arith_ev, errfmt),
241 }
242 builtins = { # Lookup
243 builtin_i.echo: io_osh.Echo(exec_opts),
244 builtin_i.shift: assign_osh.Shift(mem),
245
246 builtin_i.history: readline_osh.History(
247 readline,
248 mem,
249 errfmt,
250 mylib.Stdout(),
251 ),
252
253 builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
254 builtin_i.compadjust: completion_osh.CompAdjust(mem),
255
256 builtin_i.alias: pure_osh.Alias(aliases, errfmt),
257 builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
258 }
259
260 debug_f = util.DebugFile(sys.stderr)
261 cmd_deps = cmd_eval.Deps()
262 cmd_deps.mutable_opts = mutable_opts
263
264 search_path = executor.SearchPath(mem, exec_opts)
265
266 ext_prog = \
267 ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
268
269 cmd_deps.dumper = dev.CrashDumper('', fd_state)
270 cmd_deps.debug_f = debug_f
271
272 splitter = split.SplitContext(mem)
273
274 bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
275 parse_ctx, errfmt)
276 expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
277 errfmt)
278 tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
279 word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
280 tilde_ev, splitter, errfmt)
281 signal_safe = iolib.InitSignalSafe()
282 trap_state = trap_osh.TrapState(signal_safe)
283
284 multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
285 tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
286 multi_trace)
287 waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
288
289 cmd_deps.cflow_builtin = cmd_eval.ControlFlowBuiltin(
290 mem, exec_opts, tracer, errfmt)
291
292 cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
293 assign_builtins, arena, cmd_deps,
294 trap_state, signal_safe)
295
296 hay_state = hay_ysh.HayState()
297 shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
298 hay_state, builtins, tracer, errfmt,
299 search_path, ext_prog, waiter,
300 job_control, job_list, fd_state,
301 trap_state)
302 pure_ex = executor.PureExecutor(mem, exec_opts, mutable_opts, procs,
303 hay_state, builtins, tracer, errfmt)
304
305 assert cmd_ev.mutable_opts is not None, cmd_ev
306 prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
307
308 global_io = Obj(None, None)
309 vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
310 pure_ex, prompt_ev, global_io, tracer)
311
312 try:
313 from _devbuild.gen.help_meta import TOPICS
314 except ImportError:
315 TOPICS = None # minimal dev build
316 spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
317 splitter, comp_lookup, TOPICS,
318 errfmt)
319
320 # Add some builtins that depend on the executor!
321 complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
322 builtins[builtin_i.complete] = complete_builtin
323 builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
324
325 return cmd_ev
326
327
328def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
329 """Unit tests can evaluate code strings and then use the resulting
330 CommandEvaluator."""
331 arena = parse_ctx.arena
332 errfmt = ui.ErrorFormatter()
333
334 comp_lookup = comp_lookup or completion.Lookup()
335 mem = mem or MakeMem(arena)
336 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
337 mem.exec_opts = exec_opts
338
339 #state.InitMem(mem, {}, '0.1')
340 sh_init.InitDefaultVars(mem, [])
341 mutable_opts.InitFromEnv('')
342
343 line_reader, _ = InitLexer(code_str, arena)
344 c_parser = parse_ctx.MakeOshParser(line_reader)
345
346 cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
347 comp_lookup=comp_lookup,
348 arena=arena,
349 mem=mem,
350 aliases=aliases)
351
352 main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
353 return cmd_ev
354
355
356def InitParseContext(arena=None,
357 ysh_grammar=None,
358 aliases=None,
359 parse_opts=None,
360 do_lossless=False):
361 arena = arena or MakeArena('<test_lib>')
362
363 if aliases is None:
364 aliases = {}
365
366 mem = MakeMem(arena)
367 if parse_opts is None:
368 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
369
370 parse_ctx = parse_lib.ParseContext(arena,
371 parse_opts,
372 aliases,
373 ysh_grammar,
374 do_lossless=do_lossless)
375
376 return parse_ctx
377
378
379def InitWordParser(word_str, oil_at=False, arena=None):
380 arena = arena or MakeArena('<test_lib>')
381
382 mem = MakeMem(arena)
383 parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
384
385 # CUSTOM SETTING
386 mutable_opts.opt0_array[option_i.parse_at] = oil_at
387
388 loader = pyutil.GetResourceLoader()
389 ysh_grammar = pyutil.LoadYshGrammar(loader)
390 parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
391 line_reader, _ = InitLexer(word_str, arena)
392 c_parser = parse_ctx.MakeOshParser(line_reader)
393 # Hack
394 return c_parser.w_parser
395
396
397def InitCommandParser(code_str, arena=None):
398 arena = arena or MakeArena('<test_lib>')
399
400 loader = pyutil.GetResourceLoader()
401 ysh_grammar = pyutil.LoadYshGrammar(loader)
402
403 parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
404 line_reader, _ = InitLexer(code_str, arena)
405 c_parser = parse_ctx.MakeOshParser(line_reader)
406 return c_parser
407
408
409def SetLocalString(mem, name, s):
410 # type: (state.Mem, str, str) -> None
411 """Bind a local string."""
412 assert isinstance(s, str)
413 mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)