OILS / builtin / func_misc.py View on Github | oilshell.org

752 lines, 435 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
8
9from core import error
10from core import num
11from display import pp_value
12from display import ui
13from core import vm
14from data_lang import j8
15from frontend import match
16from frontend import typed_args
17from mycpp import mops
18from mycpp import mylib
19from mycpp.mylib import NewDict, iteritems, log, tagswitch
20from ysh import val_ops
21
22from typing import TYPE_CHECKING, Dict, List, Optional, cast
23if TYPE_CHECKING:
24 from osh import glob_
25 from osh import split
26
27_ = log
28
29
30class Object(vm._Callable):
31 """Create a value.Obj
32
33 The order of params follows JavaScript's Object.create():
34 var obj = Object(prototype, props)
35 """
36
37 def __init__(self):
38 # type: () -> None
39 pass
40
41 def Call(self, rd):
42 # type: (typed_args.Reader) -> value_t
43
44 prototype = rd.PosValue()
45 proto_loc = rd.BlamePos()
46
47 props = rd.PosDict()
48 rd.Done()
49
50 chain = None # type: Optional[Obj]
51 UP_prototype = prototype
52 with tagswitch(prototype) as case:
53 if case(value_e.Null):
54 pass
55 elif case(value_e.Obj):
56 prototype = cast(Obj, UP_prototype)
57 chain = prototype
58 else:
59 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
60 proto_loc)
61
62 return Obj(chain, props)
63
64
65class Prototype(vm._Callable):
66 """Get an object's prototype."""
67
68 def __init__(self):
69 # type: () -> None
70 pass
71
72 def Call(self, rd):
73 # type: (typed_args.Reader) -> value_t
74 obj = rd.PosObj()
75 rd.Done()
76
77 if obj.prototype is None:
78 return value.Null
79
80 return obj.prototype
81
82
83class PropView(vm._Callable):
84 """Get a Dict view of an object's properties."""
85
86 def __init__(self):
87 # type: () -> None
88 pass
89
90 def Call(self, rd):
91 # type: (typed_args.Reader) -> value_t
92 obj = rd.PosObj()
93 rd.Done()
94
95 return value.Dict(obj.d)
96
97
98class Len(vm._Callable):
99
100 def __init__(self):
101 # type: () -> None
102 pass
103
104 def Call(self, rd):
105 # type: (typed_args.Reader) -> value_t
106
107 x = rd.PosValue()
108 rd.Done()
109
110 UP_x = x
111 with tagswitch(x) as case:
112 if case(value_e.List):
113 x = cast(value.List, UP_x)
114 return num.ToBig(len(x.items))
115
116 elif case(value_e.Dict):
117 x = cast(value.Dict, UP_x)
118 return num.ToBig(len(x.d))
119
120 elif case(value_e.Str):
121 x = cast(value.Str, UP_x)
122 return num.ToBig(len(x.s))
123
124 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
125 rd.BlamePos())
126
127
128class Type(vm._Callable):
129
130 def __init__(self):
131 # type: () -> None
132 pass
133
134 def Call(self, rd):
135 # type: (typed_args.Reader) -> value_t
136
137 val = rd.PosValue()
138 rd.Done()
139
140 # TODO: assert it's not Undef, Interrupted, Slice
141 # Then return an Obj type
142 #
143 # It would be nice if they were immutable, if we didn't have to create
144 # 23-24 dicts and 23-24 Obj on startup?
145 return value.Str(ui.ValType(val))
146
147
148class Join(vm._Callable):
149 """Both free function join() and List->join() method."""
150
151 def __init__(self):
152 # type: () -> None
153 pass
154
155 def Call(self, rd):
156 # type: (typed_args.Reader) -> value_t
157
158 li = rd.PosList()
159 delim = rd.OptionalStr(default_='')
160 rd.Done()
161
162 strs = [] # type: List[str]
163 for i, el in enumerate(li):
164 strs.append(val_ops.Stringify(el, rd.LeftParenToken(), 'join() '))
165
166 return value.Str(delim.join(strs))
167
168
169class Maybe(vm._Callable):
170
171 def __init__(self):
172 # type: () -> None
173 pass
174
175 def Call(self, rd):
176 # type: (typed_args.Reader) -> value_t
177
178 val = rd.PosValue()
179 rd.Done()
180
181 if val == value.Null:
182 return value.List([])
183
184 s = val_ops.ToStr(
185 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
186 rd.LeftParenToken())
187 if len(s):
188 return value.List([val]) # use val to avoid needlessly copy
189
190 return value.List([])
191
192
193class Bool(vm._Callable):
194
195 def __init__(self):
196 # type: () -> None
197 pass
198
199 def Call(self, rd):
200 # type: (typed_args.Reader) -> value_t
201
202 val = rd.PosValue()
203 rd.Done()
204
205 return value.Bool(val_ops.ToBool(val))
206
207
208class Int(vm._Callable):
209
210 def __init__(self):
211 # type: () -> None
212 pass
213
214 def Call(self, rd):
215 # type: (typed_args.Reader) -> value_t
216
217 val = rd.PosValue()
218 rd.Done()
219
220 UP_val = val
221 with tagswitch(val) as case:
222 if case(value_e.Int):
223 return val
224
225 elif case(value_e.Bool):
226 val = cast(value.Bool, UP_val)
227 return value.Int(mops.FromBool(val.b))
228
229 elif case(value_e.Float):
230 val = cast(value.Float, UP_val)
231 ok, big_int = mops.FromFloat(val.f)
232 if ok:
233 return value.Int(big_int)
234 else:
235 raise error.Expr(
236 "Can't convert float %s to Int" %
237 pp_value.FloatString(val.f), rd.BlamePos())
238
239 elif case(value_e.Str):
240 val = cast(value.Str, UP_val)
241 if not match.LooksLikeYshInt(val.s):
242 raise error.Expr("Can't convert %s to Int" % val.s,
243 rd.BlamePos())
244
245 s = val.s.replace('_', '')
246 ok, big_int = mops.FromStr2(s)
247 if not ok:
248 raise error.Expr("Integer too big: %s" % val.s,
249 rd.BlamePos())
250
251 return value.Int(big_int)
252
253 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
254 rd.BlamePos())
255
256
257class Float(vm._Callable):
258
259 def __init__(self):
260 # type: () -> None
261 pass
262
263 def Call(self, rd):
264 # type: (typed_args.Reader) -> value_t
265
266 val = rd.PosValue()
267 rd.Done()
268
269 UP_val = val
270 with tagswitch(val) as case:
271 if case(value_e.Int):
272 val = cast(value.Int, UP_val)
273 return value.Float(mops.ToFloat(val.i))
274
275 elif case(value_e.Float):
276 return val
277
278 elif case(value_e.Str):
279 val = cast(value.Str, UP_val)
280 if not match.LooksLikeYshFloat(val.s):
281 raise error.Expr('Cannot convert %s to Float' % val.s,
282 rd.BlamePos())
283
284 return value.Float(float(val.s))
285
286 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
287 rd.BlamePos())
288
289
290class Str_(vm._Callable):
291
292 def __init__(self):
293 # type: () -> None
294 pass
295
296 def Call(self, rd):
297 # type: (typed_args.Reader) -> value_t
298
299 val = rd.PosValue()
300 rd.Done()
301
302 with tagswitch(val) as case:
303 # Avoid extra allocation
304 if case(value_e.Str):
305 return val
306 else:
307 s = val_ops.Stringify(val, rd.LeftParenToken(), 'str() ')
308 return value.Str(s)
309
310
311class List_(vm._Callable):
312
313 def __init__(self):
314 # type: () -> None
315 pass
316
317 def Call(self, rd):
318 # type: (typed_args.Reader) -> value_t
319
320 val = rd.PosValue()
321 rd.Done()
322
323 l = [] # type: List[value_t]
324 it = None # type: val_ops.Iterator
325 UP_val = val
326 with tagswitch(val) as case:
327 if case(value_e.List):
328 val = cast(value.List, UP_val)
329 it = val_ops.ListIterator(val)
330
331 elif case(value_e.Dict):
332 val = cast(value.Dict, UP_val)
333 it = val_ops.DictIterator(val)
334
335 elif case(value_e.Range):
336 val = cast(value.Range, UP_val)
337 it = val_ops.RangeIterator(val)
338
339 else:
340 raise error.TypeErr(val,
341 'list() expected Dict, List, or Range',
342 rd.BlamePos())
343
344 assert it is not None
345 while True:
346 first = it.FirstValue()
347 if first is None:
348 break
349 l.append(first)
350 it.Next()
351
352 return value.List(l)
353
354
355class DictFunc(vm._Callable):
356
357 def __init__(self):
358 # type: () -> None
359 pass
360
361 def Call(self, rd):
362 # type: (typed_args.Reader) -> value_t
363
364 val = rd.PosValue()
365 rd.Done()
366
367 UP_val = val
368 with tagswitch(val) as case:
369 if case(value_e.Dict):
370 val = cast(value.Dict, UP_val)
371 d = NewDict() # type: Dict[str, value_t]
372 for k, v in iteritems(val.d):
373 d[k] = v
374
375 return value.Dict(d)
376
377 elif case(value_e.Obj):
378 val = cast(Obj, UP_val)
379 d = NewDict()
380 for k, v in iteritems(val.d):
381 d[k] = v
382
383 return value.Dict(d)
384
385 elif case(value_e.BashAssoc):
386 val = cast(value.BashAssoc, UP_val)
387 d = NewDict()
388 for k, s in iteritems(val.d):
389 d[k] = value.Str(s)
390
391 return value.Dict(d)
392
393 elif case(value_e.Frame):
394 val = cast(value.Frame, UP_val)
395 d = NewDict()
396 for k, cell in iteritems(val.frame):
397 d[k] = cell.val
398
399 return value.Dict(d)
400
401 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
402 rd.BlamePos())
403
404
405class Runes(vm._Callable):
406
407 def __init__(self):
408 # type: () -> None
409 pass
410
411 def Call(self, rd):
412 # type: (typed_args.Reader) -> value_t
413 return value.Null
414
415
416class EncodeRunes(vm._Callable):
417
418 def __init__(self):
419 # type: () -> None
420 pass
421
422 def Call(self, rd):
423 # type: (typed_args.Reader) -> value_t
424 return value.Null
425
426
427class Bytes(vm._Callable):
428
429 def __init__(self):
430 # type: () -> None
431 pass
432
433 def Call(self, rd):
434 # type: (typed_args.Reader) -> value_t
435 return value.Null
436
437
438class EncodeBytes(vm._Callable):
439
440 def __init__(self):
441 # type: () -> None
442 pass
443
444 def Call(self, rd):
445 # type: (typed_args.Reader) -> value_t
446 return value.Null
447
448
449class Split(vm._Callable):
450
451 def __init__(self, splitter):
452 # type: (split.SplitContext) -> None
453 vm._Callable.__init__(self)
454 self.splitter = splitter
455
456 def Call(self, rd):
457 # type: (typed_args.Reader) -> value_t
458 s = rd.PosStr()
459
460 ifs = rd.OptionalStr()
461
462 rd.Done()
463
464 l = [
465 value.Str(elem)
466 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
467 ] # type: List[value_t]
468 return value.List(l)
469
470
471class FloatsEqual(vm._Callable):
472
473 def __init__(self):
474 # type: () -> None
475 pass
476
477 def Call(self, rd):
478 # type: (typed_args.Reader) -> value_t
479 left = rd.PosFloat()
480 right = rd.PosFloat()
481 rd.Done()
482
483 return value.Bool(left == right)
484
485
486class Glob(vm._Callable):
487
488 def __init__(self, globber):
489 # type: (glob_.Globber) -> None
490 vm._Callable.__init__(self)
491 self.globber = globber
492
493 def Call(self, rd):
494 # type: (typed_args.Reader) -> value_t
495 s = rd.PosStr()
496 rd.Done()
497
498 out = [] # type: List[str]
499 self.globber._Glob(s, out)
500
501 l = [value.Str(elem) for elem in out] # type: List[value_t]
502 return value.List(l)
503
504
505class ToJson8(vm._Callable):
506
507 def __init__(self, is_j8):
508 # type: (bool) -> None
509 self.is_j8 = is_j8
510
511 def Call(self, rd):
512 # type: (typed_args.Reader) -> value_t
513
514 val = rd.PosValue()
515 space = mops.BigTruncate(rd.NamedInt('space', 0))
516 rd.Done()
517
518 # Convert from external JS-like API to internal API.
519 if space <= 0:
520 indent = -1
521 else:
522 indent = space
523
524 buf = mylib.BufWriter()
525 try:
526 if self.is_j8:
527 j8.PrintMessage(val, buf, indent)
528 else:
529 j8.PrintJsonMessage(val, buf, indent)
530 except error.Encode as e:
531 # status code 4 is special, for encode/decode errors.
532 raise error.Structured(4, e.Message(), rd.LeftParenToken())
533
534 return value.Str(buf.getvalue())
535
536
537class FromJson8(vm._Callable):
538
539 def __init__(self, is_j8):
540 # type: (bool) -> None
541 self.is_j8 = is_j8
542
543 def Call(self, rd):
544 # type: (typed_args.Reader) -> value_t
545
546 s = rd.PosStr()
547 rd.Done()
548
549 p = j8.Parser(s, self.is_j8)
550 try:
551 val = p.ParseValue()
552 except error.Decode as e:
553 # Right now I'm not exposing the original string, because that
554 # could lead to a memory leak in the _error Dict.
555 # The message quotes part of the string, and we could improve
556 # that. We could have a substring with context.
557 props = {
558 'start_pos': num.ToBig(e.start_pos),
559 'end_pos': num.ToBig(e.end_pos),
560 } # type: Dict[str, value_t]
561 # status code 4 is special, for encode/decode errors.
562 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
563
564 return val
565
566
567class BashArrayToSparse(vm._Callable):
568 """
569 value.BashArray -> value.SparseArray, for testing
570 """
571
572 def __init__(self):
573 # type: () -> None
574 pass
575
576 def Call(self, rd):
577 # type: (typed_args.Reader) -> value_t
578
579 strs = rd.PosBashArray()
580 rd.Done()
581
582 d = {} # type: Dict[mops.BigInt, str]
583 max_index = mops.MINUS_ONE # max index for empty array
584 for i, s in enumerate(strs):
585 if s is not None:
586 big_i = mops.IntWiden(i)
587 d[big_i] = s
588 if mops.Greater(big_i, max_index):
589 max_index = big_i
590
591 return value.SparseArray(d, max_index)
592
593
594class SparseOp(vm._Callable):
595 """
596 All ops on value.SparseArray, for testing performance
597 """
598
599 def __init__(self):
600 # type: () -> None
601 pass
602
603 def Call(self, rd):
604 # type: (typed_args.Reader) -> value_t
605
606 sp = rd.PosSparseArray()
607 d = sp.d
608 #i = mops.BigTruncate(rd.PosInt())
609 op_name = rd.PosStr()
610
611 no_str = None # type: str
612
613 if op_name == 'len': # ${#a[@]}
614 rd.Done()
615 return num.ToBig(len(d))
616
617 elif op_name == 'get': # ${a[42]}
618 index = rd.PosInt()
619 rd.Done()
620
621 s = d.get(index)
622 if s is None:
623 return value.Null
624 else:
625 return value.Str(s)
626
627 elif op_name == 'set': # a[42]=foo
628 index = rd.PosInt()
629 s = rd.PosStr()
630 rd.Done()
631
632 d[index] = s
633
634 if mops.Greater(index, sp.max_index):
635 sp.max_index = index
636
637 return value.Int(mops.ZERO)
638
639 elif op_name == 'unset': # unset 'a[1]'
640 index = rd.PosInt()
641 rd.Done()
642
643 mylib.dict_erase(d, index)
644
645 max_index = mops.MINUS_ONE # Note: this works if d is not empty
646 for i1 in d:
647 if mops.Greater(i1, max_index): # i1 > max_index
648 max_index = i1
649 sp.max_index = max_index
650
651 return value.Int(mops.ZERO)
652
653 elif op_name == 'subst': # "${a[@]}"
654 # Algorithm to expand a Dict[BigInt, Str]
655 #
656 # 1. Copy the integer keys into a new List
657 # 2. Sort them in numeric order
658 # 3. Create a List[str] that's the same size as the keys
659 # 4. Loop through sorted keys, look up value, and populate list
660 #
661 # There is another possible algorithm:
662 #
663 # 1. Copy the VALUES into a new list
664 # 2. Somehow sort them by the CORRESPONDING key, which depends on
665 # Slab<> POSITION. I think this does not fit within the
666 # std::sort() model. I think we would have to write a little custom
667 # sort algorithm.
668
669 keys = d.keys()
670 mylib.BigIntSort(keys)
671 # Pre-allocate
672 items = [no_str] * len(d) # type: List[str]
673 j = 0
674 for i in keys:
675 s = d.get(i)
676 assert s is not None
677 items[j] = s
678 j += 1
679 return value.BashArray(items)
680
681 elif op_name == 'keys': # "${!a[@]}"
682 keys = d.keys()
683 mylib.BigIntSort(keys)
684 items = [mops.ToStr(k) for k in keys]
685
686 # TODO: return SparseArray
687 return value.BashArray(items)
688
689 elif op_name == 'slice': # "${a[@]:0:5}"
690 start = rd.PosInt()
691 end = rd.PosInt()
692 rd.Done()
693
694 n = mops.BigTruncate(mops.Sub(end, start))
695 #log('start %d - end %d', start.i, end.i)
696
697 # Pre-allocate
698 items2 = [no_str] * n # type: List[str]
699
700 # Iterate from start to end. Note that this algorithm is
701 # theoretically slower than bash in the case where the array is
702 # sparse (in the part selected by the slice)
703 #
704 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
705 # elements, OSH will iterate through 999 integers and do 999 dict
706 # lookups, while bash will follow 3 pointers.
707 #
708 # However, in practice, I think iterating through integers is
709 # cheap.
710
711 j = 0
712 i = start
713 while mops.Greater(end, i): # i < end
714 s = d.get(i)
715 #log('s %s', s)
716 if s is not None:
717 items2[j] = s
718 j += 1
719
720 i = mops.Add(i, mops.ONE) # i += 1
721
722 # TODO: return SparseArray
723 return value.BashArray(items2)
724
725 elif op_name == 'append': # a+=(x y)
726 strs = rd.PosBashArray()
727
728 # TODO: We can maintain the max index in the value.SparseArray(),
729 # so that it's O(1) to append rather than O(n)
730 # - Update on 'set' is O(1)
731 # - Update on 'unset' is potentially O(n)
732
733 if 0:
734 max_index = mops.MINUS_ONE # Note: this works for empty arrays
735 for i1 in d:
736 if mops.Greater(i1, max_index): # i1 > max_index
737 max_index = i1
738 else:
739 max_index = sp.max_index
740
741 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
742 for s in strs:
743 d[i2] = s
744 i2 = mops.Add(i2, mops.ONE) # i2 += 1
745
746 # sp.max_index += len(strs)
747 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
748 return value.Int(mops.ZERO)
749
750 else:
751 print('Invalid SparseArray operation %r' % op_name)
752 return value.Int(mops.ZERO)