| 1 | #!/usr/bin/env python2
|
| 2 | # Copyright 2016 Andy Chu. All rights reserved.
|
| 3 | # Licensed under the Apache License, Version 2.0 (the "License");
|
| 4 | # you may not use this file except in compliance with the License.
|
| 5 | # You may obtain a copy of the License at
|
| 6 | #
|
| 7 | # http://www.apache.org/licenses/LICENSE-2.0
|
| 8 | """
|
| 9 | test_lib.py - Functions for testing.
|
| 10 | """
|
| 11 |
|
| 12 | import string
|
| 13 | import sys
|
| 14 |
|
| 15 | from _devbuild.gen.option_asdl import builtin_i, option_i
|
| 16 | from _devbuild.gen.runtime_asdl import cmd_value, scope_e
|
| 17 | from _devbuild.gen.syntax_asdl import loc, source, SourceLine, Token
|
| 18 | from _devbuild.gen.value_asdl import value, Obj
|
| 19 | from asdl import pybase
|
| 20 | from builtin import assign_osh
|
| 21 | from builtin import completion_osh
|
| 22 | from builtin import hay_ysh
|
| 23 | from builtin import io_osh
|
| 24 | from builtin import pure_osh
|
| 25 | from builtin import readline_osh
|
| 26 | from builtin import trap_osh
|
| 27 | from core import alloc
|
| 28 | from core import completion
|
| 29 | from core import dev
|
| 30 | from core import executor
|
| 31 | from core import main_loop
|
| 32 | from core import optview
|
| 33 | from core import process
|
| 34 | from core import pyutil
|
| 35 | from core import sh_init
|
| 36 | from core import state
|
| 37 | from display import ui
|
| 38 | from core import util
|
| 39 | from core import vm
|
| 40 | from frontend import lexer
|
| 41 | from frontend import location
|
| 42 | from frontend import parse_lib
|
| 43 | from frontend import reader
|
| 44 | from osh import cmd_eval
|
| 45 | from osh import prompt
|
| 46 | from osh import sh_expr_eval
|
| 47 | from osh import split
|
| 48 | from osh import word_eval
|
| 49 | from ysh import expr_eval
|
| 50 | from mycpp import iolib
|
| 51 | from mycpp import mylib
|
| 52 |
|
| 53 | import posix_ as posix
|
| 54 |
|
| 55 |
|
| 56 | def MakeMem(arena):
|
| 57 | return state.Mem('', [], arena, [], {})
|
| 58 |
|
| 59 |
|
| 60 | def MakeBuiltinArgv(argv):
|
| 61 | return cmd_value.Argv(argv, [loc.Missing] * len(argv), False, None, None)
|
| 62 |
|
| 63 |
|
| 64 | def FakeTok(id_, val):
|
| 65 | # type: (int, str) -> Token
|
| 66 | src = source.Interactive
|
| 67 | line = SourceLine(1, val, src)
|
| 68 | return Token(id_, len(val), 0, line, None)
|
| 69 |
|
| 70 |
|
| 71 | def PrintableString(s):
|
| 72 | """For pretty-printing in tests."""
|
| 73 | if all(c in string.printable for c in s):
|
| 74 | return s
|
| 75 | return repr(s)
|
| 76 |
|
| 77 |
|
| 78 | def TokensEqual(left, right):
|
| 79 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
| 80 |
|
| 81 | if left.id != right.id:
|
| 82 | return False
|
| 83 |
|
| 84 | if left.line is not None:
|
| 85 | left_str = lexer.TokenVal(left)
|
| 86 | else:
|
| 87 | left_str = None
|
| 88 |
|
| 89 | if right.line is not None:
|
| 90 | right_str = lexer.TokenVal(right)
|
| 91 | else:
|
| 92 | right_str = None
|
| 93 |
|
| 94 | # Better error message sometimes:
|
| 95 | #assert left_str == right_str, '%r != %r' % (left_str, right_str)
|
| 96 | return left_str == right_str
|
| 97 |
|
| 98 |
|
| 99 | def TokenWordsEqual(left, right):
|
| 100 | # Ignoring location in CompoundObj.__eq__ now, but we might want this later.
|
| 101 | return TokensEqual(left.token, right.token)
|
| 102 | #return left == right
|
| 103 |
|
| 104 |
|
| 105 | def AsdlEqual(left, right):
|
| 106 | """Check if generated ASDL instances are equal.
|
| 107 |
|
| 108 | We don't use equality in the actual code, so this is relegated to
|
| 109 | test_lib.
|
| 110 | """
|
| 111 | if left is None and right is None:
|
| 112 | return True
|
| 113 |
|
| 114 | if isinstance(left, (int, str, bool, pybase.SimpleObj)):
|
| 115 | return left == right
|
| 116 |
|
| 117 | if isinstance(left, list):
|
| 118 | if len(left) != len(right):
|
| 119 | return False
|
| 120 | for a, b in zip(left, right):
|
| 121 | if not AsdlEqual(a, b):
|
| 122 | return False
|
| 123 | return True
|
| 124 |
|
| 125 | if isinstance(left, pybase.CompoundObj):
|
| 126 | if left.tag() != right.tag():
|
| 127 | return False
|
| 128 |
|
| 129 | field_names = left.__slots__ # hack for now
|
| 130 | for name in field_names:
|
| 131 | # Special case: we are not testing locations right now.
|
| 132 | if name == 'span_id':
|
| 133 | continue
|
| 134 | a = getattr(left, name)
|
| 135 | b = getattr(right, name)
|
| 136 | if not AsdlEqual(a, b):
|
| 137 | return False
|
| 138 |
|
| 139 | return True
|
| 140 |
|
| 141 | raise AssertionError(left)
|
| 142 |
|
| 143 |
|
| 144 | def AssertAsdlEqual(test, left, right):
|
| 145 | test.assertTrue(AsdlEqual(left, right),
|
| 146 | 'Expected %s, got %s' % (left, right))
|
| 147 |
|
| 148 |
|
| 149 | def MakeArena(source_name):
|
| 150 | arena = alloc.Arena(save_tokens=True)
|
| 151 | arena.PushSource(source.MainFile(source_name))
|
| 152 | return arena
|
| 153 |
|
| 154 |
|
| 155 | def InitLineLexer(s, arena):
|
| 156 | line_lexer = lexer.LineLexer(arena)
|
| 157 | src = source.Interactive
|
| 158 | line_lexer.Reset(SourceLine(1, s, src), 0)
|
| 159 | return line_lexer
|
| 160 |
|
| 161 |
|
| 162 | def InitLexer(s, arena):
|
| 163 | """For tests only."""
|
| 164 | line_lexer = lexer.LineLexer(arena)
|
| 165 | line_reader = reader.StringLineReader(s, arena)
|
| 166 | lx = lexer.Lexer(line_lexer, line_reader)
|
| 167 | return line_reader, lx
|
| 168 |
|
| 169 |
|
| 170 | def InitWordEvaluator(exec_opts=None):
|
| 171 | arena = MakeArena('<InitWordEvaluator>')
|
| 172 | mem = MakeMem(arena)
|
| 173 |
|
| 174 | if exec_opts is None:
|
| 175 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
|
| 176 | mem.exec_opts = exec_opts # circular dep
|
| 177 | sh_init.InitDefaultVars(mem, [])
|
| 178 | mutable_opts.InitFromEnv('')
|
| 179 | else:
|
| 180 | mutable_opts = None
|
| 181 |
|
| 182 | cmd_deps = cmd_eval.Deps()
|
| 183 | cmd_deps.trap_nodes = []
|
| 184 |
|
| 185 | splitter = split.SplitContext(mem)
|
| 186 | errfmt = ui.ErrorFormatter()
|
| 187 |
|
| 188 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
| 189 | ev = word_eval.CompletionWordEvaluator(mem, exec_opts, mutable_opts,
|
| 190 | tilde_ev, splitter, errfmt)
|
| 191 | return ev
|
| 192 |
|
| 193 |
|
| 194 | def InitCommandEvaluator(parse_ctx=None,
|
| 195 | comp_lookup=None,
|
| 196 | arena=None,
|
| 197 | mem=None,
|
| 198 | aliases=None,
|
| 199 | ext_prog=None):
|
| 200 |
|
| 201 | opt0_array = state.InitOpts()
|
| 202 | opt_stacks = [None] * option_i.ARRAY_SIZE
|
| 203 | if parse_ctx:
|
| 204 | arena = parse_ctx.arena
|
| 205 | else:
|
| 206 | parse_ctx = InitParseContext()
|
| 207 |
|
| 208 | mem = mem or MakeMem(arena)
|
| 209 | exec_opts = optview.Exec(opt0_array, opt_stacks)
|
| 210 | mutable_opts = state.MutableOpts(mem, {}, opt0_array, opt_stacks, None)
|
| 211 | mem.exec_opts = exec_opts
|
| 212 | #state.InitMem(mem, {}, '0.1')
|
| 213 | sh_init.InitDefaultVars(mem, [])
|
| 214 | mutable_opts.InitFromEnv('')
|
| 215 |
|
| 216 | # No 'readline' in the tests.
|
| 217 |
|
| 218 | errfmt = ui.ErrorFormatter()
|
| 219 | job_control = process.JobControl()
|
| 220 | job_list = process.JobList()
|
| 221 | fd_state = process.FdState(errfmt, job_control, job_list, None, None, None,
|
| 222 | exec_opts)
|
| 223 | aliases = {} if aliases is None else aliases
|
| 224 | procs = state.Procs(mem)
|
| 225 | methods = {}
|
| 226 |
|
| 227 | compopt_state = completion.OptionState()
|
| 228 | comp_lookup = comp_lookup or completion.Lookup()
|
| 229 |
|
| 230 | readline = None # simulate not having it
|
| 231 |
|
| 232 | arith_ev = sh_expr_eval.ArithEvaluator(mem, exec_opts, mutable_opts,
|
| 233 | parse_ctx, errfmt)
|
| 234 | new_var = assign_osh.NewVar(mem, procs, exec_opts, arith_ev, errfmt)
|
| 235 | assign_builtins = {
|
| 236 | builtin_i.declare: new_var,
|
| 237 | builtin_i.typeset: new_var,
|
| 238 | builtin_i.local: new_var,
|
| 239 | builtin_i.export_: assign_osh.Export(mem, arith_ev, errfmt),
|
| 240 | builtin_i.readonly: assign_osh.Readonly(mem, arith_ev, errfmt),
|
| 241 | }
|
| 242 | builtins = { # Lookup
|
| 243 | builtin_i.echo: io_osh.Echo(exec_opts),
|
| 244 | builtin_i.shift: assign_osh.Shift(mem),
|
| 245 |
|
| 246 | builtin_i.history: readline_osh.History(
|
| 247 | readline,
|
| 248 | mem,
|
| 249 | errfmt,
|
| 250 | mylib.Stdout(),
|
| 251 | ),
|
| 252 |
|
| 253 | builtin_i.compopt: completion_osh.CompOpt(compopt_state, errfmt),
|
| 254 | builtin_i.compadjust: completion_osh.CompAdjust(mem),
|
| 255 |
|
| 256 | builtin_i.alias: pure_osh.Alias(aliases, errfmt),
|
| 257 | builtin_i.unalias: pure_osh.UnAlias(aliases, errfmt),
|
| 258 | }
|
| 259 |
|
| 260 | debug_f = util.DebugFile(sys.stderr)
|
| 261 | cmd_deps = cmd_eval.Deps()
|
| 262 | cmd_deps.mutable_opts = mutable_opts
|
| 263 |
|
| 264 | # Set $PATH, to make core/process_test.py more realistic
|
| 265 | state.SetGlobalString(mem, 'PATH', ':'.join(executor.DEFAULT_PATH))
|
| 266 | search_path = executor.SearchPath(mem, exec_opts)
|
| 267 |
|
| 268 | ext_prog = \
|
| 269 | ext_prog or process.ExternalProgram('', fd_state, errfmt, debug_f)
|
| 270 |
|
| 271 | cmd_deps.dumper = dev.CrashDumper('', fd_state)
|
| 272 | cmd_deps.debug_f = debug_f
|
| 273 |
|
| 274 | splitter = split.SplitContext(mem)
|
| 275 |
|
| 276 | bool_ev = sh_expr_eval.BoolEvaluator(mem, exec_opts, mutable_opts,
|
| 277 | parse_ctx, errfmt)
|
| 278 | expr_ev = expr_eval.ExprEvaluator(mem, mutable_opts, methods, splitter,
|
| 279 | errfmt)
|
| 280 | tilde_ev = word_eval.TildeEvaluator(mem, exec_opts)
|
| 281 | word_ev = word_eval.NormalWordEvaluator(mem, exec_opts, mutable_opts,
|
| 282 | tilde_ev, splitter, errfmt)
|
| 283 | signal_safe = iolib.InitSignalSafe()
|
| 284 | trap_state = trap_osh.TrapState(signal_safe)
|
| 285 |
|
| 286 | multi_trace = dev.MultiTracer(posix.getpid(), '', '', '', fd_state)
|
| 287 | tracer = dev.Tracer(parse_ctx, exec_opts, mutable_opts, mem, debug_f,
|
| 288 | multi_trace)
|
| 289 | waiter = process.Waiter(job_list, exec_opts, trap_state, tracer)
|
| 290 |
|
| 291 | cmd_deps.cflow_builtin = cmd_eval.ControlFlowBuiltin(
|
| 292 | mem, exec_opts, tracer, errfmt)
|
| 293 |
|
| 294 | cmd_ev = cmd_eval.CommandEvaluator(mem, exec_opts, errfmt, procs,
|
| 295 | assign_builtins, arena, cmd_deps,
|
| 296 | trap_state, signal_safe)
|
| 297 |
|
| 298 | hay_state = hay_ysh.HayState()
|
| 299 | shell_ex = executor.ShellExecutor(mem, exec_opts, mutable_opts, procs,
|
| 300 | hay_state, builtins, tracer, errfmt,
|
| 301 | search_path, ext_prog, waiter,
|
| 302 | job_control, job_list, fd_state,
|
| 303 | trap_state)
|
| 304 | pure_ex = executor.PureExecutor(mem, exec_opts, mutable_opts, procs,
|
| 305 | hay_state, builtins, tracer, errfmt)
|
| 306 |
|
| 307 | assert cmd_ev.mutable_opts is not None, cmd_ev
|
| 308 | prompt_ev = prompt.Evaluator('osh', '0.0.0', parse_ctx, mem)
|
| 309 |
|
| 310 | global_io = Obj(None, None)
|
| 311 | vm.InitCircularDeps(arith_ev, bool_ev, expr_ev, word_ev, cmd_ev, shell_ex,
|
| 312 | pure_ex, prompt_ev, global_io, tracer)
|
| 313 |
|
| 314 | try:
|
| 315 | from _devbuild.gen.help_meta import TOPICS
|
| 316 | except ImportError:
|
| 317 | TOPICS = None # minimal dev build
|
| 318 | spec_builder = completion_osh.SpecBuilder(cmd_ev, parse_ctx, word_ev,
|
| 319 | splitter, comp_lookup, TOPICS,
|
| 320 | errfmt)
|
| 321 |
|
| 322 | # Add some builtins that depend on the executor!
|
| 323 | complete_builtin = completion_osh.Complete(spec_builder, comp_lookup)
|
| 324 | builtins[builtin_i.complete] = complete_builtin
|
| 325 | builtins[builtin_i.compgen] = completion_osh.CompGen(spec_builder)
|
| 326 |
|
| 327 | return cmd_ev
|
| 328 |
|
| 329 |
|
| 330 | def EvalCode(code_str, parse_ctx, comp_lookup=None, mem=None, aliases=None):
|
| 331 | """Unit tests can evaluate code strings and then use the resulting
|
| 332 | CommandEvaluator."""
|
| 333 | arena = parse_ctx.arena
|
| 334 | errfmt = ui.ErrorFormatter()
|
| 335 |
|
| 336 | comp_lookup = comp_lookup or completion.Lookup()
|
| 337 | mem = mem or MakeMem(arena)
|
| 338 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
|
| 339 | mem.exec_opts = exec_opts
|
| 340 |
|
| 341 | #state.InitMem(mem, {}, '0.1')
|
| 342 | sh_init.InitDefaultVars(mem, [])
|
| 343 | mutable_opts.InitFromEnv('')
|
| 344 |
|
| 345 | line_reader, _ = InitLexer(code_str, arena)
|
| 346 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 347 |
|
| 348 | cmd_ev = InitCommandEvaluator(parse_ctx=parse_ctx,
|
| 349 | comp_lookup=comp_lookup,
|
| 350 | arena=arena,
|
| 351 | mem=mem,
|
| 352 | aliases=aliases)
|
| 353 |
|
| 354 | main_loop.Batch(cmd_ev, c_parser, errfmt) # Parse and execute!
|
| 355 | return cmd_ev
|
| 356 |
|
| 357 |
|
| 358 | def InitParseContext(arena=None,
|
| 359 | ysh_grammar=None,
|
| 360 | aliases=None,
|
| 361 | parse_opts=None,
|
| 362 | do_lossless=False):
|
| 363 | arena = arena or MakeArena('<test_lib>')
|
| 364 |
|
| 365 | if aliases is None:
|
| 366 | aliases = {}
|
| 367 |
|
| 368 | mem = MakeMem(arena)
|
| 369 | if parse_opts is None:
|
| 370 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
|
| 371 |
|
| 372 | parse_ctx = parse_lib.ParseContext(arena,
|
| 373 | parse_opts,
|
| 374 | aliases,
|
| 375 | ysh_grammar,
|
| 376 | do_lossless=do_lossless)
|
| 377 |
|
| 378 | return parse_ctx
|
| 379 |
|
| 380 |
|
| 381 | def InitWordParser(word_str, oil_at=False, arena=None):
|
| 382 | arena = arena or MakeArena('<test_lib>')
|
| 383 |
|
| 384 | mem = MakeMem(arena)
|
| 385 | parse_opts, exec_opts, mutable_opts = state.MakeOpts(mem, {}, None)
|
| 386 |
|
| 387 | # CUSTOM SETTING
|
| 388 | mutable_opts.opt0_array[option_i.parse_at] = oil_at
|
| 389 |
|
| 390 | loader = pyutil.GetResourceLoader()
|
| 391 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
| 392 | parse_ctx = parse_lib.ParseContext(arena, parse_opts, {}, ysh_grammar)
|
| 393 | line_reader, _ = InitLexer(word_str, arena)
|
| 394 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 395 | # Hack
|
| 396 | return c_parser.w_parser
|
| 397 |
|
| 398 |
|
| 399 | def InitCommandParser(code_str, arena=None):
|
| 400 | arena = arena or MakeArena('<test_lib>')
|
| 401 |
|
| 402 | loader = pyutil.GetResourceLoader()
|
| 403 | ysh_grammar = pyutil.LoadYshGrammar(loader)
|
| 404 |
|
| 405 | parse_ctx = InitParseContext(arena=arena, ysh_grammar=ysh_grammar)
|
| 406 | line_reader, _ = InitLexer(code_str, arena)
|
| 407 | c_parser = parse_ctx.MakeOshParser(line_reader)
|
| 408 | return c_parser
|
| 409 |
|
| 410 |
|
| 411 | def SetLocalString(mem, name, s):
|
| 412 | # type: (state.Mem, str, str) -> None
|
| 413 | """Bind a local string."""
|
| 414 | assert isinstance(s, str)
|
| 415 | mem.SetNamed(location.LName(name), value.Str(s), scope_e.LocalOnly)
|