OILS / builtin / func_misc.py View on Github | oilshell.org

742 lines, 428 significant
1#!/usr/bin/env python2
2"""
3func_misc.py
4"""
5from __future__ import print_function
6
7from _devbuild.gen.value_asdl import (value, value_e, value_t, value_str, Obj)
8
9from core import error
10from core import num
11from display import pp_value
12from display import ui
13from core import vm
14from data_lang import j8
15from frontend import match
16from frontend import typed_args
17from mycpp import mops
18from mycpp import mylib
19from mycpp.mylib import NewDict, iteritems, log, tagswitch
20from ysh import val_ops
21
22from typing import TYPE_CHECKING, Dict, List, Optional, cast
23if TYPE_CHECKING:
24 from osh import glob_
25 from osh import split
26
27_ = log
28
29
30class Object(vm._Callable):
31 """Create a value.Obj
32
33 The order of params follows JavaScript's Object.create():
34 var obj = Object(prototype, props)
35 """
36
37 def __init__(self):
38 # type: () -> None
39 pass
40
41 def Call(self, rd):
42 # type: (typed_args.Reader) -> value_t
43
44 prototype = rd.PosValue()
45 props = rd.PosDict()
46 rd.Done()
47
48 chain = None # type: Optional[Obj]
49 UP_prototype = prototype
50 with tagswitch(prototype) as case:
51 if case(value_e.Null):
52 pass
53 elif case(value_e.Obj):
54 prototype = cast(Obj, UP_prototype)
55 chain = prototype
56 else:
57 raise error.TypeErr(prototype, 'Object() expected Obj or Null',
58 rd.BlamePos())
59
60 return Obj(chain, props)
61
62
63class Prototype(vm._Callable):
64 """Get an object's prototype."""
65
66 def __init__(self):
67 # type: () -> None
68 pass
69
70 def Call(self, rd):
71 # type: (typed_args.Reader) -> value_t
72 obj = rd.PosObj()
73 rd.Done()
74
75 if obj.prototype is None:
76 return value.Null
77
78 return obj.prototype
79
80
81class PropView(vm._Callable):
82 """Get a Dict view of an object's properties."""
83
84 def __init__(self):
85 # type: () -> None
86 pass
87
88 def Call(self, rd):
89 # type: (typed_args.Reader) -> value_t
90 obj = rd.PosObj()
91 rd.Done()
92
93 return value.Dict(obj.d)
94
95
96class Len(vm._Callable):
97
98 def __init__(self):
99 # type: () -> None
100 pass
101
102 def Call(self, rd):
103 # type: (typed_args.Reader) -> value_t
104
105 x = rd.PosValue()
106 rd.Done()
107
108 UP_x = x
109 with tagswitch(x) as case:
110 if case(value_e.List):
111 x = cast(value.List, UP_x)
112 return num.ToBig(len(x.items))
113
114 elif case(value_e.Dict):
115 x = cast(value.Dict, UP_x)
116 return num.ToBig(len(x.d))
117
118 elif case(value_e.Str):
119 x = cast(value.Str, UP_x)
120 return num.ToBig(len(x.s))
121
122 raise error.TypeErr(x, 'len() expected Str, List, or Dict',
123 rd.BlamePos())
124
125
126class Type(vm._Callable):
127
128 def __init__(self):
129 # type: () -> None
130 pass
131
132 def Call(self, rd):
133 # type: (typed_args.Reader) -> value_t
134
135 val = rd.PosValue()
136 rd.Done()
137
138 # TODO: assert it's not Undef, Interrupted, Slice
139 # Then return an Obj type
140 #
141 # It would be nice if they were immutable, if we didn't have to create
142 # 23-24 dicts and 23-24 Obj on startup?
143 return value.Str(ui.ValType(val))
144
145
146class Join(vm._Callable):
147 """Both free function join() and List->join() method."""
148
149 def __init__(self):
150 # type: () -> None
151 pass
152
153 def Call(self, rd):
154 # type: (typed_args.Reader) -> value_t
155
156 li = rd.PosList()
157 delim = rd.OptionalStr(default_='')
158 rd.Done()
159
160 strs = [] # type: List[str]
161 for i, el in enumerate(li):
162 strs.append(val_ops.Stringify(el, rd.LeftParenToken(), 'join() '))
163
164 return value.Str(delim.join(strs))
165
166
167class Maybe(vm._Callable):
168
169 def __init__(self):
170 # type: () -> None
171 pass
172
173 def Call(self, rd):
174 # type: (typed_args.Reader) -> value_t
175
176 val = rd.PosValue()
177 rd.Done()
178
179 if val == value.Null:
180 return value.List([])
181
182 s = val_ops.ToStr(
183 val, 'maybe() expected Str, but got %s' % value_str(val.tag()),
184 rd.LeftParenToken())
185 if len(s):
186 return value.List([val]) # use val to avoid needlessly copy
187
188 return value.List([])
189
190
191class Bool(vm._Callable):
192
193 def __init__(self):
194 # type: () -> None
195 pass
196
197 def Call(self, rd):
198 # type: (typed_args.Reader) -> value_t
199
200 val = rd.PosValue()
201 rd.Done()
202
203 return value.Bool(val_ops.ToBool(val))
204
205
206class Int(vm._Callable):
207
208 def __init__(self):
209 # type: () -> None
210 pass
211
212 def Call(self, rd):
213 # type: (typed_args.Reader) -> value_t
214
215 val = rd.PosValue()
216 rd.Done()
217
218 UP_val = val
219 with tagswitch(val) as case:
220 if case(value_e.Int):
221 return val
222
223 elif case(value_e.Bool):
224 val = cast(value.Bool, UP_val)
225 return value.Int(mops.FromBool(val.b))
226
227 elif case(value_e.Float):
228 val = cast(value.Float, UP_val)
229 ok, big_int = mops.FromFloat(val.f)
230 if ok:
231 return value.Int(big_int)
232 else:
233 raise error.Expr(
234 "Can't convert float %s to Int" %
235 pp_value.FloatString(val.f), rd.BlamePos())
236
237 elif case(value_e.Str):
238 val = cast(value.Str, UP_val)
239 if not match.LooksLikeYshInt(val.s):
240 raise error.Expr("Can't convert %s to Int" % val.s,
241 rd.BlamePos())
242
243 s = val.s.replace('_', '')
244 ok, big_int = mops.FromStr2(s)
245 if not ok:
246 raise error.Expr("Integer too big: %s" % val.s,
247 rd.BlamePos())
248
249 return value.Int(big_int)
250
251 raise error.TypeErr(val, 'int() expected Bool, Int, Float, or Str',
252 rd.BlamePos())
253
254
255class Float(vm._Callable):
256
257 def __init__(self):
258 # type: () -> None
259 pass
260
261 def Call(self, rd):
262 # type: (typed_args.Reader) -> value_t
263
264 val = rd.PosValue()
265 rd.Done()
266
267 UP_val = val
268 with tagswitch(val) as case:
269 if case(value_e.Int):
270 val = cast(value.Int, UP_val)
271 return value.Float(mops.ToFloat(val.i))
272
273 elif case(value_e.Float):
274 return val
275
276 elif case(value_e.Str):
277 val = cast(value.Str, UP_val)
278 if not match.LooksLikeYshFloat(val.s):
279 raise error.Expr('Cannot convert %s to Float' % val.s,
280 rd.BlamePos())
281
282 return value.Float(float(val.s))
283
284 raise error.TypeErr(val, 'float() expected Int, Float, or Str',
285 rd.BlamePos())
286
287
288class Str_(vm._Callable):
289
290 def __init__(self):
291 # type: () -> None
292 pass
293
294 def Call(self, rd):
295 # type: (typed_args.Reader) -> value_t
296
297 val = rd.PosValue()
298 rd.Done()
299
300 with tagswitch(val) as case:
301 # Avoid extra allocation
302 if case(value_e.Str):
303 return val
304 else:
305 s = val_ops.Stringify(val, rd.LeftParenToken(), 'str() ')
306 return value.Str(s)
307
308
309class List_(vm._Callable):
310
311 def __init__(self):
312 # type: () -> None
313 pass
314
315 def Call(self, rd):
316 # type: (typed_args.Reader) -> value_t
317
318 val = rd.PosValue()
319 rd.Done()
320
321 l = [] # type: List[value_t]
322 it = None # type: val_ops.Iterator
323 UP_val = val
324 with tagswitch(val) as case:
325 if case(value_e.List):
326 val = cast(value.List, UP_val)
327 it = val_ops.ListIterator(val)
328
329 elif case(value_e.Dict):
330 val = cast(value.Dict, UP_val)
331 it = val_ops.DictIterator(val)
332
333 elif case(value_e.Range):
334 val = cast(value.Range, UP_val)
335 it = val_ops.RangeIterator(val)
336
337 else:
338 raise error.TypeErr(val,
339 'list() expected Dict, List, or Range',
340 rd.BlamePos())
341
342 assert it is not None
343 while True:
344 first = it.FirstValue()
345 if first is None:
346 break
347 l.append(first)
348 it.Next()
349
350 return value.List(l)
351
352
353class DictFunc(vm._Callable):
354
355 def __init__(self):
356 # type: () -> None
357 pass
358
359 def Call(self, rd):
360 # type: (typed_args.Reader) -> value_t
361
362 val = rd.PosValue()
363 rd.Done()
364
365 UP_val = val
366 with tagswitch(val) as case:
367 if case(value_e.Dict):
368 d = NewDict() # type: Dict[str, value_t]
369 val = cast(value.Dict, UP_val)
370 for k, v in iteritems(val.d):
371 d[k] = v
372
373 return value.Dict(d)
374
375 elif case(value_e.Obj):
376 d = NewDict()
377 val = cast(Obj, UP_val)
378 for k, v in iteritems(val.d):
379 d[k] = v
380
381 return value.Dict(d)
382
383 elif case(value_e.BashAssoc):
384 d = NewDict()
385 val = cast(value.BashAssoc, UP_val)
386 for k, s in iteritems(val.d):
387 d[k] = value.Str(s)
388
389 return value.Dict(d)
390
391 raise error.TypeErr(val, 'dict() expected Dict, Obj, or BashAssoc',
392 rd.BlamePos())
393
394
395class Runes(vm._Callable):
396
397 def __init__(self):
398 # type: () -> None
399 pass
400
401 def Call(self, rd):
402 # type: (typed_args.Reader) -> value_t
403 return value.Null
404
405
406class EncodeRunes(vm._Callable):
407
408 def __init__(self):
409 # type: () -> None
410 pass
411
412 def Call(self, rd):
413 # type: (typed_args.Reader) -> value_t
414 return value.Null
415
416
417class Bytes(vm._Callable):
418
419 def __init__(self):
420 # type: () -> None
421 pass
422
423 def Call(self, rd):
424 # type: (typed_args.Reader) -> value_t
425 return value.Null
426
427
428class EncodeBytes(vm._Callable):
429
430 def __init__(self):
431 # type: () -> None
432 pass
433
434 def Call(self, rd):
435 # type: (typed_args.Reader) -> value_t
436 return value.Null
437
438
439class Split(vm._Callable):
440
441 def __init__(self, splitter):
442 # type: (split.SplitContext) -> None
443 vm._Callable.__init__(self)
444 self.splitter = splitter
445
446 def Call(self, rd):
447 # type: (typed_args.Reader) -> value_t
448 s = rd.PosStr()
449
450 ifs = rd.OptionalStr()
451
452 rd.Done()
453
454 l = [
455 value.Str(elem)
456 for elem in self.splitter.SplitForWordEval(s, ifs=ifs)
457 ] # type: List[value_t]
458 return value.List(l)
459
460
461class FloatsEqual(vm._Callable):
462
463 def __init__(self):
464 # type: () -> None
465 pass
466
467 def Call(self, rd):
468 # type: (typed_args.Reader) -> value_t
469 left = rd.PosFloat()
470 right = rd.PosFloat()
471 rd.Done()
472
473 return value.Bool(left == right)
474
475
476class Glob(vm._Callable):
477
478 def __init__(self, globber):
479 # type: (glob_.Globber) -> None
480 vm._Callable.__init__(self)
481 self.globber = globber
482
483 def Call(self, rd):
484 # type: (typed_args.Reader) -> value_t
485 s = rd.PosStr()
486 rd.Done()
487
488 out = [] # type: List[str]
489 self.globber._Glob(s, out)
490
491 l = [value.Str(elem) for elem in out] # type: List[value_t]
492 return value.List(l)
493
494
495class ToJson8(vm._Callable):
496
497 def __init__(self, is_j8):
498 # type: (bool) -> None
499 self.is_j8 = is_j8
500
501 def Call(self, rd):
502 # type: (typed_args.Reader) -> value_t
503
504 val = rd.PosValue()
505 space = mops.BigTruncate(rd.NamedInt('space', 0))
506 rd.Done()
507
508 # Convert from external JS-like API to internal API.
509 if space <= 0:
510 indent = -1
511 else:
512 indent = space
513
514 buf = mylib.BufWriter()
515 try:
516 if self.is_j8:
517 j8.PrintMessage(val, buf, indent)
518 else:
519 j8.PrintJsonMessage(val, buf, indent)
520 except error.Encode as e:
521 # status code 4 is special, for encode/decode errors.
522 raise error.Structured(4, e.Message(), rd.LeftParenToken())
523
524 return value.Str(buf.getvalue())
525
526
527class FromJson8(vm._Callable):
528
529 def __init__(self, is_j8):
530 # type: (bool) -> None
531 self.is_j8 = is_j8
532
533 def Call(self, rd):
534 # type: (typed_args.Reader) -> value_t
535
536 s = rd.PosStr()
537 rd.Done()
538
539 p = j8.Parser(s, self.is_j8)
540 try:
541 val = p.ParseValue()
542 except error.Decode as e:
543 # Right now I'm not exposing the original string, because that
544 # could lead to a memory leak in the _error Dict.
545 # The message quotes part of the string, and we could improve
546 # that. We could have a substring with context.
547 props = {
548 'start_pos': num.ToBig(e.start_pos),
549 'end_pos': num.ToBig(e.end_pos),
550 } # type: Dict[str, value_t]
551 # status code 4 is special, for encode/decode errors.
552 raise error.Structured(4, e.Message(), rd.LeftParenToken(), props)
553
554 return val
555
556
557class BashArrayToSparse(vm._Callable):
558 """
559 value.BashArray -> value.SparseArray, for testing
560 """
561
562 def __init__(self):
563 # type: () -> None
564 pass
565
566 def Call(self, rd):
567 # type: (typed_args.Reader) -> value_t
568
569 strs = rd.PosBashArray()
570 rd.Done()
571
572 d = {} # type: Dict[mops.BigInt, str]
573 max_index = mops.MINUS_ONE # max index for empty array
574 for i, s in enumerate(strs):
575 if s is not None:
576 big_i = mops.IntWiden(i)
577 d[big_i] = s
578 if mops.Greater(big_i, max_index):
579 max_index = big_i
580
581 return value.SparseArray(d, max_index)
582
583
584class SparseOp(vm._Callable):
585 """
586 All ops on value.SparseArray, for testing performance
587 """
588
589 def __init__(self):
590 # type: () -> None
591 pass
592
593 def Call(self, rd):
594 # type: (typed_args.Reader) -> value_t
595
596 sp = rd.PosSparseArray()
597 d = sp.d
598 #i = mops.BigTruncate(rd.PosInt())
599 op_name = rd.PosStr()
600
601 no_str = None # type: str
602
603 if op_name == 'len': # ${#a[@]}
604 rd.Done()
605 return num.ToBig(len(d))
606
607 elif op_name == 'get': # ${a[42]}
608 index = rd.PosInt()
609 rd.Done()
610
611 s = d.get(index)
612 if s is None:
613 return value.Null
614 else:
615 return value.Str(s)
616
617 elif op_name == 'set': # a[42]=foo
618 index = rd.PosInt()
619 s = rd.PosStr()
620 rd.Done()
621
622 d[index] = s
623
624 if mops.Greater(index, sp.max_index):
625 sp.max_index = index
626
627 return value.Int(mops.ZERO)
628
629 elif op_name == 'unset': # unset 'a[1]'
630 index = rd.PosInt()
631 rd.Done()
632
633 mylib.dict_erase(d, index)
634
635 max_index = mops.MINUS_ONE # Note: this works if d is not empty
636 for i1 in d:
637 if mops.Greater(i1, max_index): # i1 > max_index
638 max_index = i1
639 sp.max_index = max_index
640
641 return value.Int(mops.ZERO)
642
643 elif op_name == 'subst': # "${a[@]}"
644 # Algorithm to expand a Dict[BigInt, Str]
645 #
646 # 1. Copy the integer keys into a new List
647 # 2. Sort them in numeric order
648 # 3. Create a List[str] that's the same size as the keys
649 # 4. Loop through sorted keys, look up value, and populate list
650 #
651 # There is another possible algorithm:
652 #
653 # 1. Copy the VALUES into a new list
654 # 2. Somehow sort them by the CORRESPONDING key, which depends on
655 # Slab<> POSITION. I think this does not fit within the
656 # std::sort() model. I think we would have to write a little custom
657 # sort algorithm.
658
659 keys = d.keys()
660 mylib.BigIntSort(keys)
661 # Pre-allocate
662 items = [no_str] * len(d) # type: List[str]
663 j = 0
664 for i in keys:
665 s = d.get(i)
666 assert s is not None
667 items[j] = s
668 j += 1
669 return value.BashArray(items)
670
671 elif op_name == 'keys': # "${!a[@]}"
672 keys = d.keys()
673 mylib.BigIntSort(keys)
674 items = [mops.ToStr(k) for k in keys]
675
676 # TODO: return SparseArray
677 return value.BashArray(items)
678
679 elif op_name == 'slice': # "${a[@]:0:5}"
680 start = rd.PosInt()
681 end = rd.PosInt()
682 rd.Done()
683
684 n = mops.BigTruncate(mops.Sub(end, start))
685 #log('start %d - end %d', start.i, end.i)
686
687 # Pre-allocate
688 items2 = [no_str] * n # type: List[str]
689
690 # Iterate from start to end. Note that this algorithm is
691 # theoretically slower than bash in the case where the array is
692 # sparse (in the part selected by the slice)
693 #
694 # e.g. if you do ${a[@]:1:1000} e.g. to SHIFT, and there are only 3
695 # elements, OSH will iterate through 999 integers and do 999 dict
696 # lookups, while bash will follow 3 pointers.
697 #
698 # However, in practice, I think iterating through integers is
699 # cheap.
700
701 j = 0
702 i = start
703 while mops.Greater(end, i): # i < end
704 s = d.get(i)
705 #log('s %s', s)
706 if s is not None:
707 items2[j] = s
708 j += 1
709
710 i = mops.Add(i, mops.ONE) # i += 1
711
712 # TODO: return SparseArray
713 return value.BashArray(items2)
714
715 elif op_name == 'append': # a+=(x y)
716 strs = rd.PosBashArray()
717
718 # TODO: We can maintain the max index in the value.SparseArray(),
719 # so that it's O(1) to append rather than O(n)
720 # - Update on 'set' is O(1)
721 # - Update on 'unset' is potentially O(n)
722
723 if 0:
724 max_index = mops.MINUS_ONE # Note: this works for empty arrays
725 for i1 in d:
726 if mops.Greater(i1, max_index): # i1 > max_index
727 max_index = i1
728 else:
729 max_index = sp.max_index
730
731 i2 = mops.Add(max_index, mops.ONE) # i2 = max_index + 1
732 for s in strs:
733 d[i2] = s
734 i2 = mops.Add(i2, mops.ONE) # i2 += 1
735
736 # sp.max_index += len(strs)
737 sp.max_index = mops.Add(sp.max_index, mops.IntWiden(len(strs)))
738 return value.Int(mops.ZERO)
739
740 else:
741 print('Invalid SparseArray operation %r' % op_name)
742 return value.Int(mops.ZERO)