OILS / data_lang / htm8.py View on Github | oils.pub

955 lines, 374 significant
1"""data_lang/htm8.py
2
3TODO
4
5API:
6- Get rid of AttrValueLexer - this should be in the TagLexer
7 - this also means that unquoted values can be more similar
8 - We can use a single lexer mode for everything inside <>
9 - the SPACE is the only difference
10- Deprecate tag_lexer.GetTagName() in favor of lx.CanonicalTagName() or
11 _LiteralTagName()
12- UTF-8 check, like JSON8
13- re2c
14 - port lexer, which will fix static typing issues
15 - the abstraction needs to support submatch?
16 - for finding the end of a tag, etc.?
17
18- LexError and ParseError need details
19 - harmonize with data_lang/j8.py, which uses error.Decode(msg, ...,
20 cur_line_num)
21
22- Copy all errors into doc/ref/chap-errors.md
23 - This helps understand the language
24
25- Update doc/htm8.md
26- list of Algorithms:
27 - lex just the top level
28 - lex both levels
29 - and match tags - this is the level for value.Htm8Frag?
30 - convert to XML!
31 - lazy selection by tag, or attr (id= and class=)
32 - lazy selection by CSS selector expression
33 - convert to DOMTree
34 - sed-like replacement of DOM Tree or element
35 - untrusted HTML filter, e.g. like StackOverflow / Reddit
36 - this is Safe HTM8
37 - should have a zero alloc way to support this, with good errors?
38 - I think most of them silently strip data
39"""
40
41import re
42
43from typing import Dict, List, Tuple, Optional, IO, Iterator, Any
44
45from _devbuild.gen.htm8_asdl import (h8_id, h8_id_t, h8_tag_id, h8_tag_id_t,
46 h8_tag_id_str, attr_name, attr_name_t,
47 attr_value_e, attr_value_t, h8_val_id)
48from doctools.util import log
49
50
51class LexError(Exception):
52 """
53 Examples of lex errors:
54
55 - h8_id.Invalid, like <> or &&
56 - Unclosed <!-- <? <![CDATA[ <script> <style>
57 """
58
59 def __init__(self, s, start_pos):
60 # type: (str, int) -> None
61 self.s = s
62 self.start_pos = start_pos
63
64 def __str__(self):
65 # type: () -> str
66 return '(LexError %r)' % (self.s[self.start_pos:self.start_pos + 20])
67
68
69def _FindLineNum(s, error_pos):
70 # type: (str, int) -> int
71 current_pos = 0
72 line_num = 1
73 while True:
74 newline_pos = s.find('\n', current_pos)
75 #log('current = %d, N %d, line %d', current_pos, newline_pos, line_num)
76
77 if newline_pos == -1: # this is the last line
78 return line_num
79 if newline_pos >= error_pos:
80 return line_num
81 line_num += 1
82 current_pos = newline_pos + 1
83
84
85class ParseError(Exception):
86 """
87 Examples of parse errors
88
89 - unbalanced tag structure
90 - ul_table.py errors
91 """
92
93 def __init__(self, msg, s=None, start_pos=-1):
94 # type: (str, Optional[str], int) -> None
95 self.msg = msg
96 self.s = s
97 self.start_pos = start_pos
98
99 def __str__(self):
100 # type: () -> str
101 if self.s is not None:
102 assert self.start_pos != -1, self.start_pos
103 snippet = (self.s[self.start_pos:self.start_pos + 20])
104
105 line_num = _FindLineNum(self.s, self.start_pos)
106 else:
107 snippet = ''
108 line_num = -1
109 msg = 'line %d: %r %r' % (line_num, self.msg, snippet)
110 return msg
111
112
113class Output(object):
114 """Takes an underlying input buffer and an output file. Maintains a
115 position in the input buffer.
116
117 Print FROM the input or print new text to the output.
118 """
119
120 def __init__(self, s, f, left_pos=0, right_pos=-1):
121 # type: (str, IO[str], int, int) -> None
122 self.s = s
123 self.f = f
124 self.pos = left_pos
125 self.right_pos = len(s) if right_pos == -1 else right_pos
126
127 def SkipTo(self, pos):
128 # type: (int) -> None
129 """Skip to a position."""
130 self.pos = pos
131
132 def PrintUntil(self, pos):
133 # type: (int) -> None
134 """Print until a position."""
135 piece = self.s[self.pos:pos]
136 self.f.write(piece)
137 self.pos = pos
138
139 def PrintTheRest(self):
140 # type: () -> None
141 """Print until the end of the string."""
142 self.PrintUntil(self.right_pos)
143
144 def Print(self, s):
145 # type: (str) -> None
146 """Print text to the underlying buffer."""
147 self.f.write(s)
148
149
150def MakeLexer(rules):
151 return [(re.compile(pat, re.VERBOSE), i) for (pat, i) in rules]
152
153
154#
155# Eggex
156#
157# Tag = / ~['>']+ /
158
159# Is this valid? A single character?
160# Tag = / ~'>'* /
161
162# Maybe better: / [NOT '>']+/
163# capital letters not allowed there?
164#
165# But then this is confusing:
166# / [NOT ~digit]+/
167#
168# / [NOT digit] / is [^\d]
169# / ~digit / is \D
170#
171# Or maybe:
172#
173# / [~ digit]+ /
174# / [~ '>']+ /
175# / [NOT '>']+ /
176
177# End = / '</' Tag '>' /
178# StartEnd = / '<' Tag '/>' /
179# Start = / '<' Tag '>' /
180#
181# EntityRef = / '&' dot{* N} ';' /
182
183# Tag name, or attribute name
184# colon is used in XML
185
186# https://www.w3.org/TR/xml/#NT-Name
187# Hm there is a lot of unicode stuff. We are simplifying parsing
188
189_NAME = r'[a-zA-Z][a-zA-Z0-9:_\-]*' # must start with letter
190
191CHAR_LEX = [
192 # Characters
193 # https://www.w3.org/TR/xml/#sec-references
194 (r'&\# [0-9]+ ;', h8_id.DecChar),
195 (r'&\# x[0-9a-fA-F]+ ;', h8_id.HexChar),
196 (r'& %s ;' % _NAME, h8_id.CharEntity),
197 # Allow unquoted, and quoted
198 (r'&', h8_id.BadAmpersand),
199]
200
201HTM8_LEX = CHAR_LEX + [
202 # TODO: CommentBegin, ProcessingBegin, CDataBegin could have an additional
203 # action associated with them? The ending substring
204 (r'<!--', h8_id.CommentBegin),
205
206 # Processing instruction are used for the XML header:
207 # <?xml version="1.0" encoding="UTF-8"?>
208 # They are technically XML-only, but in HTML5, they are another kind of
209 # comment:
210 #
211 # https://developer.mozilla.org/en-US/docs/Web/API/ProcessingInstruction
212 #
213 (r'<\?', h8_id.ProcessingBegin),
214 # Not necessary in HTML5, but occurs in XML
215 (r'<!\[CDATA\[', h8_id.CDataBegin), # <![CDATA[
216
217 # Markup declarations
218 # - In HTML5, there is only <!DOCTYPE html>
219 # - XML has 4 more declarations: <!ELEMENT ...> ATTLIST ENTITY NOTATION
220 # - these seem to be part of DTD
221 # - it's useful to skip these, and be able to parse the rest of the document
222 # - Note: < is allowed?
223 (r'<! [^>\x00]+ >', h8_id.Decl),
224
225 # Tags
226 # Notes:
227 # - We look for a valid tag name, but we don't validate attributes.
228 # That's done in the tag lexer.
229 # - We don't allow leading whitespace
230 (r'</ (%s) >' % _NAME, h8_id.EndTag),
231 # self-closing <br/> comes before StartTag
232 # could/should these be collapsed into one rule?
233 (r'< (%s) [^>\x00]* />' % _NAME, h8_id.StartEndTag), # end </a>
234 (r'< (%s) [^>\x00]* >' % _NAME, h8_id.StartTag), # start <a>
235
236 # HTML5 allows unescaped > in raw data, but < is not allowed.
237 # https://stackoverflow.com/questions/10462348/right-angle-bracket-in-html
238 #
239 # - My early blog has THREE errors when disallowing >
240 # - So do some .wwz files
241 (r'[^&<>\x00]+', h8_id.RawData),
242 (r'>', h8_id.BadGreaterThan),
243 # NUL is the end, an accomodation for re2c. Like we do in frontend/match.
244 (r'\x00', h8_id.EndOfStream),
245 # < is an error
246 (r'.', h8_id.Invalid),
247]
248
249# Old notes:
250#
251# Non-greedy matches are regular and can be matched in linear time
252# with RE2.
253#
254# https://news.ycombinator.com/item?id=27099798
255#
256
257# This person tried to do it with a regex:
258#
259# https://skeptric.com/html-comment-regexp/index.html
260
261# . is any char except newline
262# https://re2c.org/manual/manual_c.html
263
264# Discarded options
265#(r'<!-- .*? -->', h8_id.Comment),
266
267# Hack from Claude: \s\S instead of re.DOTALL. I don't like this
268#(r'<!-- [\s\S]*? -->', h8_id.Comment),
269#(r'<!-- (?:.|[\n])*? -->', h8_id.Comment),
270
271HTM8_LEX_COMPILED = MakeLexer(HTM8_LEX)
272
273
274class Lexer(object):
275
276 def __init__(self, s, left_pos=0, right_pos=-1, no_special_tags=False):
277 # type: (str, int, int, bool) -> None
278 self.s = s
279 self.pos = left_pos
280 self.right_pos = len(s) if right_pos == -1 else right_pos
281 self.no_special_tags = no_special_tags
282
283 # string -> compiled regex pattern object
284 self.cache = {} # type: Dict[str, Any]
285
286 # either </script> or </style> - we search until we see that
287 self.search_state = None # type: Optional[str]
288
289 # Position of tag name, if applicable
290 # - Set after you get a StartTag, EndTag, or StartEndTag
291 # - Unset on other tags
292 self.tag_pos_left = -1
293 self.tag_pos_right = -1
294
295 def _Read(self):
296 # type: () -> Tuple[h8_id_t, int]
297 if self.pos == self.right_pos:
298 return h8_id.EndOfStream, self.pos
299
300 assert self.pos < self.right_pos, self.pos
301
302 if self.search_state is not None and not self.no_special_tags:
303 # TODO: case-insensitive search for </SCRIPT> <SCRipt> ?
304 #
305 # Another strategy: enter a mode where we find ONLY the end tag
306 # regex, and any data that's not <, and then check the canonical
307 # tag name for 'script' or 'style'.
308 pos = self.s.find(self.search_state, self.pos)
309 if pos == -1:
310 # unterminated <script> or <style>
311 raise LexError(self.s, self.pos)
312 self.search_state = None
313 # beginning
314 return h8_id.HtmlCData, pos
315
316 # Find the first match.
317 # Note: frontend/match.py uses _LongestMatch(), which is different!
318 # TODO: reconcile them. This lexer should be expressible in re2c.
319
320 for pat, tok_id in HTM8_LEX_COMPILED:
321 m = pat.match(self.s, self.pos)
322 if m:
323 if tok_id in (h8_id.StartTag, h8_id.EndTag, h8_id.StartEndTag):
324 self.tag_pos_left = m.start(1)
325 self.tag_pos_right = m.end(1)
326 else:
327 # Reset state
328 self.tag_pos_left = -1
329 self.tag_pos_right = -1
330
331 if tok_id == h8_id.CommentBegin:
332 pos = self.s.find('-->', self.pos)
333 if pos == -1:
334 # unterminated <!--
335 raise LexError(self.s, self.pos)
336 return h8_id.Comment, pos + 3 # -->
337
338 if tok_id == h8_id.ProcessingBegin:
339 pos = self.s.find('?>', self.pos)
340 if pos == -1:
341 # unterminated <?
342 raise LexError(self.s, self.pos)
343 return h8_id.Processing, pos + 2 # ?>
344
345 if tok_id == h8_id.CDataBegin:
346 pos = self.s.find(']]>', self.pos)
347 if pos == -1:
348 # unterminated <![CDATA[
349 raise LexError(self.s, self.pos)
350 return h8_id.CData, pos + 3 # ]]>
351
352 if tok_id == h8_id.StartTag:
353 # TODO: reduce allocations
354 if (self.TagNameEquals('script') or
355 self.TagNameEquals('style')):
356 # <SCRipt a=b> -> </SCRipt>
357 self.search_state = '</' + self._LiteralTagName() + '>'
358
359 return tok_id, m.end()
360 else:
361 raise AssertionError('h8_id.Invalid rule should have matched')
362
363 def TagNamePos(self):
364 """The right position of the tag pos"""
365 assert self.tag_pos_right != -1, self.tag_pos_right
366 return self.tag_pos_right
367
368 def TagNameEquals(self, expected):
369 # type: (str) -> bool
370 assert self.tag_pos_left != -1, self.tag_pos_left
371 assert self.tag_pos_right != -1, self.tag_pos_right
372
373 # TODO: In C++, this does not need an allocation. Can we test
374 # directly?
375 return expected == self.CanonicalTagName()
376
377 def _LiteralTagName(self):
378 # type: () -> str
379 assert self.tag_pos_left != -1, self.tag_pos_left
380 assert self.tag_pos_right != -1, self.tag_pos_right
381
382 return self.s[self.tag_pos_left:self.tag_pos_right]
383
384 def CanonicalTagName(self):
385 # type: () -> str
386 tag_name = self._LiteralTagName()
387 # Most tags are already lower case, so avoid allocation with this conditional
388 # TODO: this could go in the mycpp runtime?
389 if tag_name.islower():
390 return tag_name
391 else:
392 return tag_name.lower()
393
394 def Read(self):
395 # type: () -> Tuple[h8_id_t, int]
396 tok_id, end_pos = self._Read()
397 self.pos = end_pos # advance
398 return tok_id, end_pos
399
400 def LookAhead(self, regex):
401 # type: (str) -> bool
402 """
403 Currently used for ul_table.py. But taking a dynamic regex string is
404 not the right interface.
405 """
406 # Cache the regex compilation. This could also be LookAheadFor(THEAD)
407 # or something.
408 pat = self.cache.get(regex)
409 if pat is None:
410 pat = re.compile(regex)
411 self.cache[regex] = pat
412
413 m = pat.match(self.s, self.pos)
414 return m is not None
415
416
417A_NAME_LEX = [
418 # Leading whitespace is required, to separate attributes.
419 #
420 # If the = is not present, then we set the lexer in a state for
421 # attr_value_e.Missing.
422 (r'\s+ (%s) \s* (=)? \s*' % _NAME, attr_name.Ok),
423 # unexpected EOF
424
425 # The closing > or /> is treated as end of stream, and it's not an error.
426 (r'\s* /? >', attr_name.Done),
427
428 # NUL should not be possible, because the top-level
429
430 # e.g. < is an error
431 (r'.', attr_name.Invalid),
432]
433
434A_NAME_LEX_COMPILED = MakeLexer(A_NAME_LEX)
435
436# Here we just loop on regular tokens
437#
438# Examples:
439# <a href = unquoted&amp;foo >
440# <a href = unquoted&foo > # BadAmpersand is allowed I guess
441# <a href ="unquoted&foo" > # double quoted
442# <a href ='unquoted&foo' > # single quoted
443# <a href = what"foo" > # HTML5 allows this, but we could disallow it if
444# it's not common. It opens up the j"" and $"" extensions
445# <a href = what'foo' > # ditto
446#
447# Problem: <a href=foo/> - this is hard to recognize
448# Because is the unquoted value "foo/" or "foo" ?
449
450# Be very lenient - just no whitespace or special HTML chars
451# I don't think this is more lenient than HTML5, though we should check.
452#
453# Bug fix: Also disallow /
454
455# TODO: get rid of OLD copy
456_UNQUOTED_VALUE_OLD = r'''[^ \t\r\n<>&/"'\x00]*'''
457_UNQUOTED_VALUE = r'''[^ \t\r\n<>&/"'\x00]+'''
458
459# Restrictive definition, similar to _NAME
460# I was trying to capture #ble.sh and so forth
461# I also had unquoted //github.com, etc.
462
463# _UNQUOTED_VALUE = r'''[a-zA-Z0-9:_\-]+'''
464#
465# For now, I guess we live with <a href=?foo/>
466
467A_VALUE_LEX = CHAR_LEX + [
468 (r'"', h8_val_id.DoubleQuote),
469 (r"'", h8_val_id.SingleQuote),
470 (_UNQUOTED_VALUE, h8_val_id.UnquotedVal),
471
472 #(r'[ \r\n\t]', h8_id.Whitespace), # terminates unquoted values
473 #(r'[^ \r\n\t&>\x00]', h8_id.RawData),
474 #(r'[>\x00]', h8_id.EndOfStream),
475 # e.g. < is an error
476 (r'.', h8_val_id.NoMatch),
477]
478
479A_VALUE_LEX_COMPILED = MakeLexer(A_VALUE_LEX)
480
481
482class AttrLexer(object):
483 """
484 We can also invert this
485
486 Unquoted (List[h8_id] tok_ids, List[int] end_pos)
487
488 It would be nice to have a special case for the singleton, since that is
489 very common.
490
491 Simple (int tag_name_start, int tag_name_end, int attr_value_tag,
492 int value_start, int value_end)
493 This would cover many cases
494
495 The other option is to create many different events, and have AttrValueLexer
496 But I think that is annoying and overly detailed.
497
498 Operations:
499 - GetAttrRaw('foo')
500 - AllAttrsRaw()
501 - AllAttrsRawSlice()
502
503 class= query - well we can do this with Space tokens I think - we should
504 have an optimization
505 id= query - ditto, we should just have a predicate
506
507 Zero allocs:
508 tag query - TagNameEquals()
509
510 So I guess we have to write a HasClass('foo') and IdEquals('bar') on top of
511 this. Yes.
512
513 tag_lx.Reset2(...) # we should pass it the tag_name_end position
514 tag_lx.Read() -> bool # success or fail? Or Attr or Invalid
515 .AttrNameEquals('foo') -> bool # id and class query
516 .GetAttrName() -> str # for getting them all
517 .GetRawValue() -> Tuple[h8_tag_id, start, end] # just beginning and end
518 .GetValueTokens() -> Tuple[h8_tag_id, TokenList]
519 .TokenList = Tuple[List[h8_id], List[int end_pos]
520
521 You could also have
522
523 tag_lx.GetValueTokenId() -> Tuple[h8_id, end_pos]
524
525 And then you read it until it's " or ' or space ? We probably won't have
526 that use case to start.
527 """
528
529 def __init__(self, s):
530 # type: (str) -> None
531 self.s = s
532 self.tag_name_pos = -1 # Invalid
533 self.tag_end_pos = -1
534 self.pos = -1
535
536 self.name_start = -1
537 self.name_end = -1
538 self.next_value_is_missing = False
539
540 def Init(self, tag_name_pos, end_pos):
541 # type: (int, int) -> None
542 """Initialize so we can read names and values.
543
544 Example:
545 'x <a y>' # tag_name_pos=4, end_pos=6
546 'x <a>' # tag_name_pos=4, end_pos=4
547
548 The Reset() method is used to reuse instances of the AttrLexer object.
549 """
550 assert tag_name_pos >= 0, tag_name_pos
551 assert end_pos >= 0, end_pos
552
553 log('TAG NAME POS %d', tag_name_pos)
554
555 self.tag_name_pos = tag_name_pos
556 self.end_pos = end_pos
557
558 self.pos = tag_name_pos
559
560 def ReadName(self):
561 # type: () -> Tuple[attr_name_t, int, int]
562 """Reads the attribute name
563
564 EOF case:
565 <a>
566 <a >
567
568 Error case:
569 <a !>
570 <a foo=bar !>
571 """
572 for pat, a in A_NAME_LEX_COMPILED:
573 m = pat.match(self.s, self.pos)
574 if m:
575 self.pos = m.end(0) # Advance
576
577 if a == attr_name.Ok:
578 #log('%r', m.groups())
579 self.name_start = m.start(1)
580 self.name_end = m.end(1)
581 # Set state based on =
582 if m.group(2) is None:
583 self.next_value_is_missing = True
584 return attr_name.Ok, self.name_start, self.name_end
585 else:
586 # Reset state - e.g. you must call AttrNameEquals
587 self.name_start = -1
588 self.name_end = -1
589 self.next_value_is_missing = False
590
591 if a == attr_name.Invalid:
592 return attr_name.Invalid, -1, -1
593 if a == attr_name.Done:
594 return attr_name.Done, -1, -1
595 else:
596 raise AssertionError('h8_id.Invalid rule should have matched')
597
598 def _CanonicalAttrName(self):
599 # type: () -> str
600 assert self.name_start >= 0, self.name_start
601 assert self.name_end >= 0, self.name_end
602
603 attr_name = self.s[self.name_start:self.name_end]
604 if attr_name.islower():
605 return attr_name
606 else:
607 return attr_name.lower()
608
609 def AttrNameEquals(self, expected):
610 # type: (str) -> bool
611 """
612 TODO: Must call this after ReadName() ?
613 Because that can FAIL.
614 """
615 return expected == self._CanonicalAttrName()
616
617 def ReadRawValue(self):
618 # type: () -> Tuple[attr_value_t, int, int]
619 """Read the attribute value.
620
621 In general, it is escaped or "raw"
622
623 Can only be called after a SUCCESSFUL ReadName().
624 Assuming ReadName() returned a value, this should NOT fail.
625 """
626 # ReadName() invariant
627 assert self.name_start >= 0, self.name_start
628 assert self.name_end >= 0, self.name_end
629
630 self.name_start = -1
631 self.name_end = -1
632
633 if self.next_value_is_missing:
634 return attr_value_e.Missing, -1, -1
635 else:
636 # Now read " ', unquoted or empty= is valid too.
637 for pat, a in A_VALUE_LEX_COMPILED:
638 m = pat.match(self.s, self.pos)
639 if m:
640 self.pos = m.end(0) # Advance
641
642 log('m %s', m.groups())
643 if a == h8_val_id.UnquotedVal:
644 return attr_value_e.Unquoted, m.start(0), m.end(0)
645 if a == h8_val_id.DoubleQuote:
646 # TODO: read until "
647 return attr_value_e.DoubleQuoted, m.start(0), m.end(0)
648 if a == h8_val_id.SingleQuote:
649 # TODO: read until '
650 return attr_value_e.SingleQuoted, m.start(0), m.end(0)
651 if a == h8_val_id.NoMatch:
652 # <a foo = >
653 return attr_value_e.Empty, -1, -1
654 else:
655 raise AssertionError(
656 'h8_val_id.NoMatch rule should have matched')
657
658 def SkipValue(self):
659 # type: () -> None
660 # Just ignore it and return
661 self.ReadRawValue()
662
663 def ReadValueAndDecode(self):
664 # type: () -> str
665 """Read the attribute vlaue
666 """
667 # TODO: tokenize it
668 pass
669
670
671# Tag names:
672# Match <a or </a
673# Match <h2, but not <2h
674#
675# HTML 5 doesn't restrict tag names at all
676# https://html.spec.whatwg.org/#toc-syntax
677#
678# XML allows : - .
679# https://www.w3.org/TR/xml/#NT-NameChar
680
681# Namespaces for MathML, SVG
682# XLink, XML, XMLNS
683#
684# https://infra.spec.whatwg.org/#namespaces
685#
686# Allow - for td-attrs
687
688# TODO: we don't need to capture the tag name here? That's done at the top
689# level
690_TAG_RE = re.compile(r'/? \s* (%s)' % _NAME, re.VERBOSE)
691
692_TAG_LAST_RE = re.compile(r'\s* /? >', re.VERBOSE)
693
694# To match href="foo"
695# Note: in HTML5 and XML, single quoted attributes are also valid
696
697# <button disabled> is standard usage
698
699# NOTE: This used to allow whitespace around =
700# <a foo = "bar"> makes sense in XML
701# But then you also have
702# <a foo= bar> - which is TWO attributes, in HTML5
703# So the space is problematic
704
705_ATTR_RE = re.compile(
706 r'''
707\s+ # Leading whitespace is required
708(%s) # Attribute name
709(?: # Optional attribute value
710 \s* = \s* # Spaces allowed around =
711 (?:
712 " ([^>"\x00]*) " # double quoted value
713 | ' ([^>'\x00]*) ' # single quoted value
714 | (%s) # Attribute value
715 )
716)?
717''' % (_NAME, _UNQUOTED_VALUE_OLD), re.VERBOSE)
718
719
720class TagLexer(object):
721 """
722 Given a tag like <a href="..."> or <link type="..." />, the TagLexer
723 provides a few operations:
724
725 - What is the tag?
726 - Iterate through the attributes, giving (name, value_start_pos, value_end_pos)
727 """
728
729 def __init__(self, s):
730 # type: (str) -> None
731 self.s = s
732 self.start_pos = -1 # Invalid
733 self.end_pos = -1
734
735 def Reset(self, start_pos, end_pos):
736 # type: (int, int) -> None
737 """Reuse instances of this object."""
738 assert start_pos >= 0, start_pos
739 assert end_pos >= 0, end_pos
740
741 self.start_pos = start_pos
742 self.end_pos = end_pos
743
744 def WholeTagString(self):
745 # type: () -> str
746 """Return the entire tag string, e.g. <a href='foo'>"""
747 return self.s[self.start_pos:self.end_pos]
748
749 def GetTagName(self):
750 # type: () -> str
751 # First event
752 tok_id, start, end = next(self.Tokens())
753 return self.s[start:end]
754
755 def GetSpanForAttrValue(self, attr_name):
756 # type: (str) -> Tuple[int, int]
757 """
758 Used by oils_doc.py, for href shortcuts
759 """
760 # Algorithm: search for QuotedValue or UnquotedValue after AttrName
761 # TODO: Could also cache these
762
763 events = self.Tokens()
764 val = (-1, -1)
765 try:
766 while True:
767 tok_id, start, end = next(events)
768 if tok_id == h8_tag_id.AttrName:
769 name = self.s[start:end]
770 if name == attr_name:
771 # The value should come next
772 tok_id, start, end = next(events)
773 assert tok_id in (
774 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
775 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
776 val = start, end
777 break
778
779 except StopIteration:
780 pass
781 return val
782
783 def GetAttrRaw(self, attr_name):
784 # type: (str) -> Optional[str]
785 """
786 Return the value, which may be UNESCAPED.
787 """
788 start, end = self.GetSpanForAttrValue(attr_name)
789 if start == -1:
790 return None
791 return self.s[start:end]
792
793 def AllAttrsRawSlice(self):
794 # type: () -> List[Tuple[str, int, int]]
795 """
796 Get a list of pairs [('class', 3, 5), ('href', 9, 12)]
797 """
798 slices = []
799 events = self.Tokens()
800 try:
801 while True:
802 tok_id, start, end = next(events)
803 if tok_id == h8_tag_id.AttrName:
804 name = self.s[start:end]
805
806 # The value should come next
807 tok_id, start, end = next(events)
808 assert tok_id in (
809 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
810 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
811 # Note: quoted values may have &amp;
812 # We would need ANOTHER lexer to unescape them, but we
813 # don't need that for ul-table
814 slices.append((name, start, end))
815 except StopIteration:
816 pass
817 return slices
818
819 def AllAttrsRaw(self):
820 # type: () -> List[Tuple[str, str]]
821 """
822 Get a list of pairs [('class', 'foo'), ('href', '?foo=1&amp;bar=2')]
823
824 The quoted values may be escaped. We would need another lexer to
825 unescape them.
826 """
827 slices = self.AllAttrsRawSlice()
828 pairs = []
829 for name, start, end in slices:
830 pairs.append((name, self.s[start:end]))
831 return pairs
832
833 def Tokens(self):
834 # type: () -> Iterator[Tuple[h8_tag_id_t, int, int]]
835 """
836 Yields a sequence of tokens: Tag (AttrName AttrValue?)*
837
838 Where each Token is (Type, start_pos, end_pos)
839
840 Note that start and end are NOT redundant! We skip over some unwanted
841 characters.
842 """
843 m = _TAG_RE.match(self.s, self.start_pos + 1)
844 if not m:
845 raise RuntimeError("Couldn't find HTML tag in %r" %
846 self.WholeTagString())
847 yield h8_tag_id.TagName, m.start(1), m.end(1)
848
849 pos = m.end(0)
850 #log('POS %d', pos)
851
852 while True:
853 # don't search past the end
854 m = _ATTR_RE.match(self.s, pos, self.end_pos)
855 if not m:
856 #log('BREAK pos %d', pos)
857 break
858 #log('AttrName %r', m.group(1))
859
860 yield h8_tag_id.AttrName, m.start(1), m.end(1)
861
862 #log('m.groups() %r', m.groups())
863 if m.group(2) is not None:
864 # double quoted
865 yield h8_tag_id.QuotedValue, m.start(2), m.end(2)
866 elif m.group(3) is not None:
867 # single quoted - TODO: could have different token types
868 yield h8_tag_id.QuotedValue, m.start(3), m.end(3)
869 elif m.group(4) is not None:
870 yield h8_tag_id.UnquotedValue, m.start(4), m.end(4)
871 else:
872 # <button disabled>
873 end = m.end(0)
874 yield h8_tag_id.MissingValue, end, end
875
876 # Skip past the "
877 pos = m.end(0)
878
879 #log('TOK %r', self.s)
880
881 m = _TAG_LAST_RE.match(self.s, pos)
882 #log('_TAG_LAST_RE match %r', self.s[pos:])
883 if not m:
884 # Extra data at end of tag. TODO: add messages for all these.
885 raise LexError(self.s, pos)
886
887
888# This is similar but not identical to
889# " ([^>"\x00]*) " # double quoted value
890# | ' ([^>'\x00]*) ' # single quoted value
891#
892# Note: for unquoted values, & isn't allowed, and thus &amp; and &#99; and
893# &#x99; are not allowed. We could relax that?
894ATTR_VALUE_LEX = CHAR_LEX + [
895 (r'[^>&\x00]+', h8_id.RawData),
896 (r'.', h8_id.Invalid),
897]
898
899ATTR_VALUE_LEX_COMPILED = MakeLexer(ATTR_VALUE_LEX)
900
901
902class AttrValueLexer(object):
903 """
904 <a href="foo=99&amp;bar">
905 <a href='foo=99&amp;bar'>
906 <a href=unquoted>
907 """
908
909 def __init__(self, s):
910 # type: (str) -> None
911 self.s = s
912 self.start_pos = -1 # Invalid
913 self.end_pos = -1
914
915 def Reset(self, start_pos, end_pos):
916 # type: (int, int) -> None
917 """Reuse instances of this object."""
918 assert start_pos >= 0, start_pos
919 assert end_pos >= 0, end_pos
920
921 self.start_pos = start_pos
922 self.end_pos = end_pos
923
924 def NumTokens(self):
925 # type: () -> int
926 num_tokens = 0
927 pos = self.start_pos
928 for tok_id, end_pos in self.Tokens():
929 if tok_id == h8_id.Invalid:
930 raise LexError(self.s, pos)
931 pos = end_pos
932 #log('pos %d', pos)
933 num_tokens += 1
934 return num_tokens
935
936 def Tokens(self):
937 # type: () -> Iterator[Tuple[h8_id_t, int]]
938 pos = self.start_pos
939 while pos < self.end_pos:
940 # Find the first match, like above.
941 # Note: frontend/match.py uses _LongestMatch(), which is different!
942 # TODO: reconcile them. This lexer should be expressible in re2c.
943 for pat, tok_id in ATTR_VALUE_LEX_COMPILED:
944 m = pat.match(self.s, pos)
945 if m:
946 if 0:
947 tok_str = m.group(0)
948 log('token = %r', tok_str)
949
950 end_pos = m.end(0)
951 yield tok_id, end_pos
952 pos = end_pos
953 break
954 else:
955 raise AssertionError('h8_id.Invalid rule should have matched')