OILS / builtin / func_misc.py View on Github | oilshell.org

794 lines, 457 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
8
9from core import error
10from core import num
11from display import pp_value
12from display import ui
13from core import vm
14from data_lang import j8
15from frontend import match
16from frontend import typed_args
17from mycpp import mops
18from mycpp import mylib
19from mycpp.mylib import NewDict, iteritems, log, tagswitch
20from ysh import val_ops
21
22from typing import TYPE_CHECKING, Dict, List, Optional, cast
23if TYPE_CHECKING:
24 from osh import glob_
25 from osh import split
26
27_ = log
28
29
30class Object(vm._Callable):
31 """OLD API to a value.Obj
32
33 The order of params follows JavaScript's Object.create():
34 var obj = Object(prototype, props)
35 """
36
37 def __init__(self):
38 # type: () -> None
39 pass
40
41 def Call(self, rd):
42 # type: (typed_args.Reader) -> value_t
43
44 prototype = rd.PosValue()
45 proto_loc = rd.BlamePos()
46
47 props = rd.PosDict()
48 rd.Done()
49
50 chain = None # type: Optional[Obj]
51 UP_prototype = prototype
52 with tagswitch(prototype) as case:
53 if case(value_e.Null):
54 pass
55 elif case(value_e.Obj):
56 prototype = cast(Obj, UP_prototype)
57 chain = prototype
58 else:
59 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
60 proto_loc)
61
62 return Obj(chain, props)
63
64
65class Obj_call(vm._Callable):
66 """New API to create a value.Obj
67
68 It has a more natural order
69 var obj = Obj(props, prototype)
70
71 Until we have __call__, it's Obj:
72 var obj = Obj.new(props, prototype)
73 """
74
75 def __init__(self):
76 # type: () -> None
77 pass
78
79 def Call(self, rd):
80 # type: (typed_args.Reader) -> value_t
81
82 props = rd.PosDict()
83
84 prototype = rd.OptionalValue()
85 proto_loc = rd.BlamePos()
86
87 rd.Done()
88
89 chain = None # type: Optional[Obj]
90
91 if prototype is not None:
92 UP_prototype = prototype
93 with tagswitch(prototype) as case:
94 if case(value_e.Null): # Obj({}, null)
95 pass
96 elif case(value_e.Obj):
97 prototype = cast(Obj, UP_prototype)
98 chain = prototype
99 else:
100 raise error.TypeErr(prototype,
101 'Object() expected Obj or Null',
102 proto_loc)
103
104 return Obj(chain, props)
105
106
107class Prototype(vm._Callable):
108 """Get an object's prototype."""
109
110 def __init__(self):
111 # type: () -> None
112 pass
113
114 def Call(self, rd):
115 # type: (typed_args.Reader) -> value_t
116 obj = rd.PosObj()
117 rd.Done()
118
119 if obj.prototype is None:
120 return value.Null
121
122 return obj.prototype
123
124
125class PropView(vm._Callable):
126 """Get a Dict view of an object's properties."""
127
128 def __init__(self):
129 # type: () -> None
130 pass
131
132 def Call(self, rd):
133 # type: (typed_args.Reader) -> value_t
134 obj = rd.PosObj()
135 rd.Done()
136
137 return value.Dict(obj.d)
138
139
140class Len(vm._Callable):
141
142 def __init__(self):
143 # type: () -> None
144 pass
145
146 def Call(self, rd):
147 # type: (typed_args.Reader) -> value_t
148
149 x = rd.PosValue()
150 rd.Done()
151
152 UP_x = x
153 with tagswitch(x) as case:
154 if case(value_e.List):
155 x = cast(value.List, UP_x)
156 return num.ToBig(len(x.items))
157
158 elif case(value_e.Dict):
159 x = cast(value.Dict, UP_x)
160 return num.ToBig(len(x.d))
161
162 elif case(value_e.Str):
163 x = cast(value.Str, UP_x)
164 return num.ToBig(len(x.s))
165
166 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
167 rd.BlamePos())
168
169
170class Type(vm._Callable):
171
172 def __init__(self):
173 # type: () -> None
174 pass
175
176 def Call(self, rd):
177 # type: (typed_args.Reader) -> value_t
178
179 val = rd.PosValue()
180 rd.Done()
181
182 # TODO: assert it's not Undef, Interrupted, Slice
183 # Then return an Obj type
184 #
185 # It would be nice if they were immutable, if we didn't have to create
186 # 23-24 dicts and 23-24 Obj on startup?
187 return value.Str(ui.ValType(val))
188
189
190class Join(vm._Callable):
191 """Both free function join() and List->join() method."""
192
193 def __init__(self):
194 # type: () -> None
195 pass
196
197 def Call(self, rd):
198 # type: (typed_args.Reader) -> value_t
199
200 li = rd.PosList()
201 delim = rd.OptionalStr(default_='')
202 rd.Done()
203
204 strs = [] # type: List[str]
205 for i, el in enumerate(li):
206 strs.append(val_ops.Stringify(el, rd.LeftParenToken(), 'join() '))
207
208 return value.Str(delim.join(strs))
209
210
211class Maybe(vm._Callable):
212
213 def __init__(self):
214 # type: () -> None
215 pass
216
217 def Call(self, rd):
218 # type: (typed_args.Reader) -> value_t
219
220 val = rd.PosValue()
221 rd.Done()
222
223 if val == value.Null:
224 return value.List([])
225
226 s = val_ops.ToStr(
227 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
228 rd.LeftParenToken())
229 if len(s):
230 return value.List([val]) # use val to avoid needlessly copy
231
232 return value.List([])
233
234
235class Bool(vm._Callable):
236
237 def __init__(self):
238 # type: () -> None
239 pass
240
241 def Call(self, rd):
242 # type: (typed_args.Reader) -> value_t
243
244 val = rd.PosValue()
245 rd.Done()
246
247 return value.Bool(val_ops.ToBool(val))
248
249
250class Int(vm._Callable):
251
252 def __init__(self):
253 # type: () -> None
254 pass
255
256 def Call(self, rd):
257 # type: (typed_args.Reader) -> value_t
258
259 val = rd.PosValue()
260 rd.Done()
261
262 UP_val = val
263 with tagswitch(val) as case:
264 if case(value_e.Int):
265 return val
266
267 elif case(value_e.Bool):
268 val = cast(value.Bool, UP_val)
269 return value.Int(mops.FromBool(val.b))
270
271 elif case(value_e.Float):
272 val = cast(value.Float, UP_val)
273 ok, big_int = mops.FromFloat(val.f)
274 if ok:
275 return value.Int(big_int)
276 else:
277 raise error.Expr(
278 "Can't convert float %s to Int" %
279 pp_value.FloatString(val.f), rd.BlamePos())
280
281 elif case(value_e.Str):
282 val = cast(value.Str, UP_val)
283 if not match.LooksLikeYshInt(val.s):
284 raise error.Expr("Can't convert %s to Int" % val.s,
285 rd.BlamePos())
286
287 s = val.s.replace('_', '')
288 ok, big_int = mops.FromStr2(s)
289 if not ok:
290 raise error.Expr("Integer too big: %s" % val.s,
291 rd.BlamePos())
292
293 return value.Int(big_int)
294
295 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
296 rd.BlamePos())
297
298
299class Float(vm._Callable):
300
301 def __init__(self):
302 # type: () -> None
303 pass
304
305 def Call(self, rd):
306 # type: (typed_args.Reader) -> value_t
307
308 val = rd.PosValue()
309 rd.Done()
310
311 UP_val = val
312 with tagswitch(val) as case:
313 if case(value_e.Int):
314 val = cast(value.Int, UP_val)
315 return value.Float(mops.ToFloat(val.i))
316
317 elif case(value_e.Float):
318 return val
319
320 elif case(value_e.Str):
321 val = cast(value.Str, UP_val)
322 if not match.LooksLikeYshFloat(val.s):
323 raise error.Expr('Cannot convert %s to Float' % val.s,
324 rd.BlamePos())
325
326 return value.Float(float(val.s))
327
328 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
329 rd.BlamePos())
330
331
332class Str_(vm._Callable):
333
334 def __init__(self):
335 # type: () -> None
336 pass
337
338 def Call(self, rd):
339 # type: (typed_args.Reader) -> value_t
340
341 val = rd.PosValue()
342 rd.Done()
343
344 with tagswitch(val) as case:
345 # Avoid extra allocation
346 if case(value_e.Str):
347 return val
348 else:
349 s = val_ops.Stringify(val, rd.LeftParenToken(), 'str() ')
350 return value.Str(s)
351
352
353class List_(vm._Callable):
354
355 def __init__(self):
356 # type: () -> None
357 pass
358
359 def Call(self, rd):
360 # type: (typed_args.Reader) -> value_t
361
362 val = rd.PosValue()
363 rd.Done()
364
365 l = [] # type: List[value_t]
366 it = None # type: val_ops.Iterator
367 UP_val = val
368 with tagswitch(val) as case:
369 if case(value_e.List):
370 val = cast(value.List, UP_val)
371 it = val_ops.ListIterator(val)
372
373 elif case(value_e.Dict):
374 val = cast(value.Dict, UP_val)
375 it = val_ops.DictIterator(val)
376
377 elif case(value_e.Range):
378 val = cast(value.Range, UP_val)
379 it = val_ops.RangeIterator(val)
380
381 else:
382 raise error.TypeErr(val,
383 'list() expected Dict, List, or Range',
384 rd.BlamePos())
385
386 assert it is not None
387 while True:
388 first = it.FirstValue()
389 if first is None:
390 break
391 l.append(first)
392 it.Next()
393
394 return value.List(l)
395
396
397class DictFunc(vm._Callable):
398
399 def __init__(self):
400 # type: () -> None
401 pass
402
403 def Call(self, rd):
404 # type: (typed_args.Reader) -> value_t
405
406 val = rd.PosValue()
407 rd.Done()
408
409 UP_val = val
410 with tagswitch(val) as case:
411 if case(value_e.Dict):
412 val = cast(value.Dict, UP_val)
413 d = NewDict() # type: Dict[str, value_t]
414 for k, v in iteritems(val.d):
415 d[k] = v
416
417 return value.Dict(d)
418
419 elif case(value_e.Obj):
420 val = cast(Obj, UP_val)
421 d = NewDict()
422 for k, v in iteritems(val.d):
423 d[k] = v
424
425 return value.Dict(d)
426
427 elif case(value_e.BashAssoc):
428 val = cast(value.BashAssoc, UP_val)
429 d = NewDict()
430 for k, s in iteritems(val.d):
431 d[k] = value.Str(s)
432
433 return value.Dict(d)
434
435 elif case(value_e.Frame):
436 val = cast(value.Frame, UP_val)
437 d = NewDict()
438 for k, cell in iteritems(val.frame):
439 d[k] = cell.val
440
441 return value.Dict(d)
442
443 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
444 rd.BlamePos())
445
446
447class Runes(vm._Callable):
448
449 def __init__(self):
450 # type: () -> None
451 pass
452
453 def Call(self, rd):
454 # type: (typed_args.Reader) -> value_t
455 return value.Null
456
457
458class EncodeRunes(vm._Callable):
459
460 def __init__(self):
461 # type: () -> None
462 pass
463
464 def Call(self, rd):
465 # type: (typed_args.Reader) -> value_t
466 return value.Null
467
468
469class Bytes(vm._Callable):
470
471 def __init__(self):
472 # type: () -> None
473 pass
474
475 def Call(self, rd):
476 # type: (typed_args.Reader) -> value_t
477 return value.Null
478
479
480class EncodeBytes(vm._Callable):
481
482 def __init__(self):
483 # type: () -> None
484 pass
485
486 def Call(self, rd):
487 # type: (typed_args.Reader) -> value_t
488 return value.Null
489
490
491class Split(vm._Callable):
492
493 def __init__(self, splitter):
494 # type: (split.SplitContext) -> None
495 vm._Callable.__init__(self)
496 self.splitter = splitter
497
498 def Call(self, rd):
499 # type: (typed_args.Reader) -> value_t
500 s = rd.PosStr()
501
502 ifs = rd.OptionalStr()
503
504 rd.Done()
505
506 l = [
507 value.Str(elem)
508 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
509 ] # type: List[value_t]
510 return value.List(l)
511
512
513class FloatsEqual(vm._Callable):
514
515 def __init__(self):
516 # type: () -> None
517 pass
518
519 def Call(self, rd):
520 # type: (typed_args.Reader) -> value_t
521 left = rd.PosFloat()
522 right = rd.PosFloat()
523 rd.Done()
524
525 return value.Bool(left == right)
526
527
528class Glob(vm._Callable):
529
530 def __init__(self, globber):
531 # type: (glob_.Globber) -> None
532 vm._Callable.__init__(self)
533 self.globber = globber
534
535 def Call(self, rd):
536 # type: (typed_args.Reader) -> value_t
537 s = rd.PosStr()
538 rd.Done()
539
540 out = [] # type: List[str]
541 self.globber._Glob(s, out)
542
543 l = [value.Str(elem) for elem in out] # type: List[value_t]
544 return value.List(l)
545
546
547class ToJson8(vm._Callable):
548
549 def __init__(self, is_j8):
550 # type: (bool) -> None
551 self.is_j8 = is_j8
552
553 def Call(self, rd):
554 # type: (typed_args.Reader) -> value_t
555
556 val = rd.PosValue()
557 space = mops.BigTruncate(rd.NamedInt('space', 0))
558 rd.Done()
559
560 # Convert from external JS-like API to internal API.
561 if space <= 0:
562 indent = -1
563 else:
564 indent = space
565
566 buf = mylib.BufWriter()
567 try:
568 if self.is_j8:
569 j8.PrintMessage(val, buf, indent)
570 else:
571 j8.PrintJsonMessage(val, buf, indent)
572 except error.Encode as e:
573 # status code 4 is special, for encode/decode errors.
574 raise error.Structured(4, e.Message(), rd.LeftParenToken())
575
576 return value.Str(buf.getvalue())
577
578
579class FromJson8(vm._Callable):
580
581 def __init__(self, is_j8):
582 # type: (bool) -> None
583 self.is_j8 = is_j8
584
585 def Call(self, rd):
586 # type: (typed_args.Reader) -> value_t
587
588 s = rd.PosStr()
589 rd.Done()
590
591 p = j8.Parser(s, self.is_j8)
592 try:
593 val = p.ParseValue()
594 except error.Decode as e:
595 # Right now I'm not exposing the original string, because that
596 # could lead to a memory leak in the _error Dict.
597 # The message quotes part of the string, and we could improve
598 # that. We could have a substring with context.
599 props = {
600 'start_pos': num.ToBig(e.start_pos),
601 'end_pos': num.ToBig(e.end_pos),
602 } # type: Dict[str, value_t]
603 # status code 4 is special, for encode/decode errors.
604 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
605
606 return val
607
608
609class BashArrayToSparse(vm._Callable):
610 """
611 value.BashArray -> value.SparseArray, for testing
612 """
613
614 def __init__(self):
615 # type: () -> None
616 pass
617
618 def Call(self, rd):
619 # type: (typed_args.Reader) -> value_t
620
621 strs = rd.PosBashArray()
622 rd.Done()
623
624 d = {} # type: Dict[mops.BigInt, str]
625 max_index = mops.MINUS_ONE # max index for empty array
626 for i, s in enumerate(strs):
627 if s is not None:
628 big_i = mops.IntWiden(i)
629 d[big_i] = s
630 if mops.Greater(big_i, max_index):
631 max_index = big_i
632
633 return value.SparseArray(d, max_index)
634
635
636class SparseOp(vm._Callable):
637 """
638 All ops on value.SparseArray, for testing performance
639 """
640
641 def __init__(self):
642 # type: () -> None
643 pass
644
645 def Call(self, rd):
646 # type: (typed_args.Reader) -> value_t
647
648 sp = rd.PosSparseArray()
649 d = sp.d
650 #i = mops.BigTruncate(rd.PosInt())
651 op_name = rd.PosStr()
652
653 no_str = None # type: str
654
655 if op_name == 'len': # ${#a[@]}
656 rd.Done()
657 return num.ToBig(len(d))
658
659 elif op_name == 'get': # ${a[42]}
660 index = rd.PosInt()
661 rd.Done()
662
663 s = d.get(index)
664 if s is None:
665 return value.Null
666 else:
667 return value.Str(s)
668
669 elif op_name == 'set': # a[42]=foo
670 index = rd.PosInt()
671 s = rd.PosStr()
672 rd.Done()
673
674 d[index] = s
675
676 if mops.Greater(index, sp.max_index):
677 sp.max_index = index
678
679 return value.Int(mops.ZERO)
680
681 elif op_name == 'unset': # unset 'a[1]'
682 index = rd.PosInt()
683 rd.Done()
684
685 mylib.dict_erase(d, index)
686
687 max_index = mops.MINUS_ONE # Note: this works if d is not empty
688 for i1 in d:
689 if mops.Greater(i1, max_index): # i1 > max_index
690 max_index = i1
691 sp.max_index = max_index
692
693 return value.Int(mops.ZERO)
694
695 elif op_name == 'subst': # "${a[@]}"
696 # Algorithm to expand a Dict[BigInt, Str]
697 #
698 # 1. Copy the integer keys into a new List
699 # 2. Sort them in numeric order
700 # 3. Create a List[str] that's the same size as the keys
701 # 4. Loop through sorted keys, look up value, and populate list
702 #
703 # There is another possible algorithm:
704 #
705 # 1. Copy the VALUES into a new list
706 # 2. Somehow sort them by the CORRESPONDING key, which depends on
707 # Slab<> POSITION. I think this does not fit within the
708 # std::sort() model. I think we would have to write a little custom
709 # sort algorithm.
710
711 keys = d.keys()
712 mylib.BigIntSort(keys)
713 # Pre-allocate
714 items = [no_str] * len(d) # type: List[str]
715 j = 0
716 for i in keys:
717 s = d.get(i)
718 assert s is not None
719 items[j] = s
720 j += 1
721 return value.BashArray(items)
722
723 elif op_name == 'keys': # "${!a[@]}"
724 keys = d.keys()
725 mylib.BigIntSort(keys)
726 items = [mops.ToStr(k) for k in keys]
727
728 # TODO: return SparseArray
729 return value.BashArray(items)
730
731 elif op_name == 'slice': # "${a[@]:0:5}"
732 start = rd.PosInt()
733 end = rd.PosInt()
734 rd.Done()
735
736 n = mops.BigTruncate(mops.Sub(end, start))
737 #log('start %d - end %d', start.i, end.i)
738
739 # Pre-allocate
740 items2 = [no_str] * n # type: List[str]
741
742 # Iterate from start to end. Note that this algorithm is
743 # theoretically slower than bash in the case where the array is
744 # sparse (in the part selected by the slice)
745 #
746 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
747 # elements, OSH will iterate through 999 integers and do 999 dict
748 # lookups, while bash will follow 3 pointers.
749 #
750 # However, in practice, I think iterating through integers is
751 # cheap.
752
753 j = 0
754 i = start
755 while mops.Greater(end, i): # i < end
756 s = d.get(i)
757 #log('s %s', s)
758 if s is not None:
759 items2[j] = s
760 j += 1
761
762 i = mops.Add(i, mops.ONE) # i += 1
763
764 # TODO: return SparseArray
765 return value.BashArray(items2)
766
767 elif op_name == 'append': # a+=(x y)
768 strs = rd.PosBashArray()
769
770 # TODO: We can maintain the max index in the value.SparseArray(),
771 # so that it's O(1) to append rather than O(n)
772 # - Update on 'set' is O(1)
773 # - Update on 'unset' is potentially O(n)
774
775 if 0:
776 max_index = mops.MINUS_ONE # Note: this works for empty arrays
777 for i1 in d:
778 if mops.Greater(i1, max_index): # i1 > max_index
779 max_index = i1
780 else:
781 max_index = sp.max_index
782
783 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
784 for s in strs:
785 d[i2] = s
786 i2 = mops.Add(i2, mops.ONE) # i2 += 1
787
788 # sp.max_index += len(strs)
789 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
790 return value.Int(mops.ZERO)
791
792 else:
793 print('Invalid SparseArray operation %r' % op_name)
794 return value.Int(mops.ZERO)