OILS / builtin / func_misc.py View on Github | oilshell.org

737 lines, 424 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
8
9from core import error
10from core import num
11from display import pp_value
12from display import ui
13from core import vm
14from data_lang import j8
15from frontend import match
16from frontend import typed_args
17from mycpp import mops
18from mycpp import mylib
19from mycpp.mylib import NewDict, iteritems, log, tagswitch
20from ysh import val_ops
21
22from typing import TYPE_CHECKING, Dict, List, Optional, cast
23if TYPE_CHECKING:
24 from osh import glob_
25 from osh import split
26
27_ = log
28
29
30class Object(vm._Callable):
31 """Create a value.Obj
32
33 The order of params follows JavaScript's Object.create():
34 var obj = Object(prototype, props)
35 """
36
37 def __init__(self):
38 # type: () -> None
39 pass
40
41 def Call(self, rd):
42 # type: (typed_args.Reader) -> value_t
43
44 prototype = rd.PosValue()
45 props = rd.PosDict()
46 rd.Done()
47
48 chain = None # type: Optional[Obj]
49 UP_prototype = prototype
50 with tagswitch(prototype) as case:
51 if case(value_e.Null):
52 pass
53 elif case(value_e.Obj):
54 prototype = cast(Obj, UP_prototype)
55 chain = prototype
56 else:
57 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
58 rd.BlamePos())
59
60 return Obj(chain, props)
61
62
63class Prototype(vm._Callable):
64 """Get an object's prototype."""
65
66 def __init__(self):
67 # type: () -> None
68 pass
69
70 def Call(self, rd):
71 # type: (typed_args.Reader) -> value_t
72 obj = rd.PosObj()
73 rd.Done()
74
75 if obj.prototype is None:
76 return value.Null
77
78 return obj.prototype
79
80
81class PropView(vm._Callable):
82 """Get a Dict view of an object's properties."""
83
84 def __init__(self):
85 # type: () -> None
86 pass
87
88 def Call(self, rd):
89 # type: (typed_args.Reader) -> value_t
90 obj = rd.PosObj()
91 rd.Done()
92
93 return value.Dict(obj.d)
94
95
96class Len(vm._Callable):
97
98 def __init__(self):
99 # type: () -> None
100 pass
101
102 def Call(self, rd):
103 # type: (typed_args.Reader) -> value_t
104
105 x = rd.PosValue()
106 rd.Done()
107
108 UP_x = x
109 with tagswitch(x) as case:
110 if case(value_e.List):
111 x = cast(value.List, UP_x)
112 return num.ToBig(len(x.items))
113
114 elif case(value_e.Dict):
115 x = cast(value.Dict, UP_x)
116 return num.ToBig(len(x.d))
117
118 elif case(value_e.Str):
119 x = cast(value.Str, UP_x)
120 return num.ToBig(len(x.s))
121
122 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
123 rd.BlamePos())
124
125
126class Type(vm._Callable):
127
128 def __init__(self):
129 # type: () -> None
130 pass
131
132 def Call(self, rd):
133 # type: (typed_args.Reader) -> value_t
134
135 val = rd.PosValue()
136 rd.Done()
137
138 # TODO: assert it's not Undef, Interrupted, Slice
139 # Then return an Obj type
140 #
141 # It would be nice if they were immutable, if we didn't have to create
142 # 23-24 dicts and 23-24 Obj on startup?
143 return value.Str(ui.ValType(val))
144
145
146class Join(vm._Callable):
147 """Both free function join() and List->join() method."""
148
149 def __init__(self):
150 # type: () -> None
151 pass
152
153 def Call(self, rd):
154 # type: (typed_args.Reader) -> value_t
155
156 li = rd.PosList()
157 delim = rd.OptionalStr(default_='')
158 rd.Done()
159
160 strs = [] # type: List[str]
161 for i, el in enumerate(li):
162 strs.append(val_ops.Stringify(el, rd.LeftParenToken(), 'join() '))
163
164 return value.Str(delim.join(strs))
165
166
167class Maybe(vm._Callable):
168
169 def __init__(self):
170 # type: () -> None
171 pass
172
173 def Call(self, rd):
174 # type: (typed_args.Reader) -> value_t
175
176 val = rd.PosValue()
177 rd.Done()
178
179 if val == value.Null:
180 return value.List([])
181
182 s = val_ops.ToStr(
183 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
184 rd.LeftParenToken())
185 if len(s):
186 return value.List([val]) # use val to avoid needlessly copy
187
188 return value.List([])
189
190
191class Bool(vm._Callable):
192
193 def __init__(self):
194 # type: () -> None
195 pass
196
197 def Call(self, rd):
198 # type: (typed_args.Reader) -> value_t
199
200 val = rd.PosValue()
201 rd.Done()
202
203 return value.Bool(val_ops.ToBool(val))
204
205
206class Int(vm._Callable):
207
208 def __init__(self):
209 # type: () -> None
210 pass
211
212 def Call(self, rd):
213 # type: (typed_args.Reader) -> value_t
214
215 val = rd.PosValue()
216 rd.Done()
217
218 UP_val = val
219 with tagswitch(val) as case:
220 if case(value_e.Int):
221 return val
222
223 elif case(value_e.Bool):
224 val = cast(value.Bool, UP_val)
225 return value.Int(mops.FromBool(val.b))
226
227 elif case(value_e.Float):
228 val = cast(value.Float, UP_val)
229 ok, big_int = mops.FromFloat(val.f)
230 if ok:
231 return value.Int(big_int)
232 else:
233 raise error.Expr(
234 "Can't convert float %s to Int" %
235 pp_value.FloatString(val.f), rd.BlamePos())
236
237 elif case(value_e.Str):
238 val = cast(value.Str, UP_val)
239 if not match.LooksLikeYshInt(val.s):
240 raise error.Expr("Can't convert %s to Int" % val.s,
241 rd.BlamePos())
242
243 s = val.s.replace('_', '')
244 return value.Int(mops.FromStr(s))
245
246 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
247 rd.BlamePos())
248
249
250class Float(vm._Callable):
251
252 def __init__(self):
253 # type: () -> None
254 pass
255
256 def Call(self, rd):
257 # type: (typed_args.Reader) -> value_t
258
259 val = rd.PosValue()
260 rd.Done()
261
262 UP_val = val
263 with tagswitch(val) as case:
264 if case(value_e.Int):
265 val = cast(value.Int, UP_val)
266 return value.Float(mops.ToFloat(val.i))
267
268 elif case(value_e.Float):
269 return val
270
271 elif case(value_e.Str):
272 val = cast(value.Str, UP_val)
273 if not match.LooksLikeYshFloat(val.s):
274 raise error.Expr('Cannot convert %s to Float' % val.s,
275 rd.BlamePos())
276
277 return value.Float(float(val.s))
278
279 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
280 rd.BlamePos())
281
282
283class Str_(vm._Callable):
284
285 def __init__(self):
286 # type: () -> None
287 pass
288
289 def Call(self, rd):
290 # type: (typed_args.Reader) -> value_t
291
292 val = rd.PosValue()
293 rd.Done()
294
295 with tagswitch(val) as case:
296 # Avoid extra allocation
297 if case(value_e.Str):
298 return val
299 else:
300 s = val_ops.Stringify(val, rd.LeftParenToken(), 'str() ')
301 return value.Str(s)
302
303
304class List_(vm._Callable):
305
306 def __init__(self):
307 # type: () -> None
308 pass
309
310 def Call(self, rd):
311 # type: (typed_args.Reader) -> value_t
312
313 val = rd.PosValue()
314 rd.Done()
315
316 l = [] # type: List[value_t]
317 it = None # type: val_ops.Iterator
318 UP_val = val
319 with tagswitch(val) as case:
320 if case(value_e.List):
321 val = cast(value.List, UP_val)
322 it = val_ops.ListIterator(val)
323
324 elif case(value_e.Dict):
325 val = cast(value.Dict, UP_val)
326 it = val_ops.DictIterator(val)
327
328 elif case(value_e.Range):
329 val = cast(value.Range, UP_val)
330 it = val_ops.RangeIterator(val)
331
332 else:
333 raise error.TypeErr(val,
334 'list() expected Dict, List, or Range',
335 rd.BlamePos())
336
337 assert it is not None
338 while True:
339 first = it.FirstValue()
340 if first is None:
341 break
342 l.append(first)
343 it.Next()
344
345 return value.List(l)
346
347
348class DictFunc(vm._Callable):
349
350 def __init__(self):
351 # type: () -> None
352 pass
353
354 def Call(self, rd):
355 # type: (typed_args.Reader) -> value_t
356
357 val = rd.PosValue()
358 rd.Done()
359
360 UP_val = val
361 with tagswitch(val) as case:
362 if case(value_e.Dict):
363 d = NewDict() # type: Dict[str, value_t]
364 val = cast(value.Dict, UP_val)
365 for k, v in iteritems(val.d):
366 d[k] = v
367
368 return value.Dict(d)
369
370 elif case(value_e.Obj):
371 d = NewDict()
372 val = cast(Obj, UP_val)
373 for k, v in iteritems(val.d):
374 d[k] = v
375
376 return value.Dict(d)
377
378 elif case(value_e.BashAssoc):
379 d = NewDict()
380 val = cast(value.BashAssoc, UP_val)
381 for k, s in iteritems(val.d):
382 d[k] = value.Str(s)
383
384 return value.Dict(d)
385
386 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
387 rd.BlamePos())
388
389
390class Runes(vm._Callable):
391
392 def __init__(self):
393 # type: () -> None
394 pass
395
396 def Call(self, rd):
397 # type: (typed_args.Reader) -> value_t
398 return value.Null
399
400
401class EncodeRunes(vm._Callable):
402
403 def __init__(self):
404 # type: () -> None
405 pass
406
407 def Call(self, rd):
408 # type: (typed_args.Reader) -> value_t
409 return value.Null
410
411
412class Bytes(vm._Callable):
413
414 def __init__(self):
415 # type: () -> None
416 pass
417
418 def Call(self, rd):
419 # type: (typed_args.Reader) -> value_t
420 return value.Null
421
422
423class EncodeBytes(vm._Callable):
424
425 def __init__(self):
426 # type: () -> None
427 pass
428
429 def Call(self, rd):
430 # type: (typed_args.Reader) -> value_t
431 return value.Null
432
433
434class Split(vm._Callable):
435
436 def __init__(self, splitter):
437 # type: (split.SplitContext) -> None
438 vm._Callable.__init__(self)
439 self.splitter = splitter
440
441 def Call(self, rd):
442 # type: (typed_args.Reader) -> value_t
443 s = rd.PosStr()
444
445 ifs = rd.OptionalStr()
446
447 rd.Done()
448
449 l = [
450 value.Str(elem)
451 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
452 ] # type: List[value_t]
453 return value.List(l)
454
455
456class FloatsEqual(vm._Callable):
457
458 def __init__(self):
459 # type: () -> None
460 pass
461
462 def Call(self, rd):
463 # type: (typed_args.Reader) -> value_t
464 left = rd.PosFloat()
465 right = rd.PosFloat()
466 rd.Done()
467
468 return value.Bool(left == right)
469
470
471class Glob(vm._Callable):
472
473 def __init__(self, globber):
474 # type: (glob_.Globber) -> None
475 vm._Callable.__init__(self)
476 self.globber = globber
477
478 def Call(self, rd):
479 # type: (typed_args.Reader) -> value_t
480 s = rd.PosStr()
481 rd.Done()
482
483 out = [] # type: List[str]
484 self.globber._Glob(s, out)
485
486 l = [value.Str(elem) for elem in out] # type: List[value_t]
487 return value.List(l)
488
489
490class ToJson8(vm._Callable):
491
492 def __init__(self, is_j8):
493 # type: (bool) -> None
494 self.is_j8 = is_j8
495
496 def Call(self, rd):
497 # type: (typed_args.Reader) -> value_t
498
499 val = rd.PosValue()
500 space = mops.BigTruncate(rd.NamedInt('space', 0))
501 rd.Done()
502
503 # Convert from external JS-like API to internal API.
504 if space <= 0:
505 indent = -1
506 else:
507 indent = space
508
509 buf = mylib.BufWriter()
510 try:
511 if self.is_j8:
512 j8.PrintMessage(val, buf, indent)
513 else:
514 j8.PrintJsonMessage(val, buf, indent)
515 except error.Encode as e:
516 # status code 4 is special, for encode/decode errors.
517 raise error.Structured(4, e.Message(), rd.LeftParenToken())
518
519 return value.Str(buf.getvalue())
520
521
522class FromJson8(vm._Callable):
523
524 def __init__(self, is_j8):
525 # type: (bool) -> None
526 self.is_j8 = is_j8
527
528 def Call(self, rd):
529 # type: (typed_args.Reader) -> value_t
530
531 s = rd.PosStr()
532 rd.Done()
533
534 p = j8.Parser(s, self.is_j8)
535 try:
536 val = p.ParseValue()
537 except error.Decode as e:
538 # Right now I'm not exposing the original string, because that
539 # could lead to a memory leak in the _error Dict.
540 # The message quotes part of the string, and we could improve
541 # that. We could have a substring with context.
542 props = {
543 'start_pos': num.ToBig(e.start_pos),
544 'end_pos': num.ToBig(e.end_pos),
545 } # type: Dict[str, value_t]
546 # status code 4 is special, for encode/decode errors.
547 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
548
549 return val
550
551
552class BashArrayToSparse(vm._Callable):
553 """
554 value.BashArray -> value.SparseArray, for testing
555 """
556
557 def __init__(self):
558 # type: () -> None
559 pass
560
561 def Call(self, rd):
562 # type: (typed_args.Reader) -> value_t
563
564 strs = rd.PosBashArray()
565 rd.Done()
566
567 d = {} # type: Dict[mops.BigInt, str]
568 max_index = mops.MINUS_ONE # max index for empty array
569 for i, s in enumerate(strs):
570 if s is not None:
571 big_i = mops.IntWiden(i)
572 d[big_i] = s
573 if mops.Greater(big_i, max_index):
574 max_index = big_i
575
576 return value.SparseArray(d, max_index)
577
578
579class SparseOp(vm._Callable):
580 """
581 All ops on value.SparseArray, for testing performance
582 """
583
584 def __init__(self):
585 # type: () -> None
586 pass
587
588 def Call(self, rd):
589 # type: (typed_args.Reader) -> value_t
590
591 sp = rd.PosSparseArray()
592 d = sp.d
593 #i = mops.BigTruncate(rd.PosInt())
594 op_name = rd.PosStr()
595
596 no_str = None # type: str
597
598 if op_name == 'len': # ${#a[@]}
599 rd.Done()
600 return num.ToBig(len(d))
601
602 elif op_name == 'get': # ${a[42]}
603 index = rd.PosInt()
604 rd.Done()
605
606 s = d.get(index)
607 if s is None:
608 return value.Null
609 else:
610 return value.Str(s)
611
612 elif op_name == 'set': # a[42]=foo
613 index = rd.PosInt()
614 s = rd.PosStr()
615 rd.Done()
616
617 d[index] = s
618
619 if mops.Greater(index, sp.max_index):
620 sp.max_index = index
621
622 return value.Int(mops.ZERO)
623
624 elif op_name == 'unset': # unset 'a[1]'
625 index = rd.PosInt()
626 rd.Done()
627
628 mylib.dict_erase(d, index)
629
630 max_index = mops.MINUS_ONE # Note: this works if d is not empty
631 for i1 in d:
632 if mops.Greater(i1, max_index): # i1 > max_index
633 max_index = i1
634 sp.max_index = max_index
635
636 return value.Int(mops.ZERO)
637
638 elif op_name == 'subst': # "${a[@]}"
639 # Algorithm to expand a Dict[BigInt, Str]
640 #
641 # 1. Copy the integer keys into a new List
642 # 2. Sort them in numeric order
643 # 3. Create a List[str] that's the same size as the keys
644 # 4. Loop through sorted keys, look up value, and populate list
645 #
646 # There is another possible algorithm:
647 #
648 # 1. Copy the VALUES into a new list
649 # 2. Somehow sort them by the CORRESPONDING key, which depends on
650 # Slab<> POSITION. I think this does not fit within the
651 # std::sort() model. I think we would have to write a little custom
652 # sort algorithm.
653
654 keys = d.keys()
655 mylib.BigIntSort(keys)
656 # Pre-allocate
657 items = [no_str] * len(d) # type: List[str]
658 j = 0
659 for i in keys:
660 s = d.get(i)
661 assert s is not None
662 items[j] = s
663 j += 1
664 return value.BashArray(items)
665
666 elif op_name == 'keys': # "${!a[@]}"
667 keys = d.keys()
668 mylib.BigIntSort(keys)
669 items = [mops.ToStr(k) for k in keys]
670
671 # TODO: return SparseArray
672 return value.BashArray(items)
673
674 elif op_name == 'slice': # "${a[@]:0:5}"
675 start = rd.PosInt()
676 end = rd.PosInt()
677 rd.Done()
678
679 n = mops.BigTruncate(mops.Sub(end, start))
680 #log('start %d - end %d', start.i, end.i)
681
682 # Pre-allocate
683 items2 = [no_str] * n # type: List[str]
684
685 # Iterate from start to end. Note that this algorithm is
686 # theoretically slower than bash in the case where the array is
687 # sparse (in the part selected by the slice)
688 #
689 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
690 # elements, OSH will iterate through 999 integers and do 999 dict
691 # lookups, while bash will follow 3 pointers.
692 #
693 # However, in practice, I think iterating through integers is
694 # cheap.
695
696 j = 0
697 i = start
698 while mops.Greater(end, i): # i < end
699 s = d.get(i)
700 #log('s %s', s)
701 if s is not None:
702 items2[j] = s
703 j += 1
704
705 i = mops.Add(i, mops.ONE) # i += 1
706
707 # TODO: return SparseArray
708 return value.BashArray(items2)
709
710 elif op_name == 'append': # a+=(x y)
711 strs = rd.PosBashArray()
712
713 # TODO: We can maintain the max index in the value.SparseArray(),
714 # so that it's O(1) to append rather than O(n)
715 # - Update on 'set' is O(1)
716 # - Update on 'unset' is potentially O(n)
717
718 if 0:
719 max_index = mops.MINUS_ONE # Note: this works for empty arrays
720 for i1 in d:
721 if mops.Greater(i1, max_index): # i1 > max_index
722 max_index = i1
723 else:
724 max_index = sp.max_index
725
726 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
727 for s in strs:
728 d[i2] = s
729 i2 = mops.Add(i2, mops.ONE) # i2 += 1
730
731 # sp.max_index += len(strs)
732 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
733 return value.Int(mops.ZERO)
734
735 else:
736 print('Invalid SparseArray operation %r' % op_name)
737 return value.Int(mops.ZERO)