OILS / data_lang / htm8.py View on Github | oils.pub

915 lines, 351 significant
1"""data_lang/htm8.py
2
3TODO
4
5API:
6- Get rid of AttrValueLexer - this should be in the TagLexer
7 - this also means that unquoted values can be more similar
8 - We can use a single lexer mode for everything inside <>
9 - the SPACE is the only difference
10- Deprecate tag_lexer.GetTagName() in favor of lx.CanonicalTagName() or
11 _LiteralTagName()
12- UTF-8 check, like JSON8
13- re2c
14 - port lexer, which will fix static typing issues
15 - the abstraction needs to support submatch?
16 - for finding the end of a tag, etc.?
17
18- LexError and ParseError need details
19 - harmonize with data_lang/j8.py, which uses error.Decode(msg, ...,
20 cur_line_num)
21
22- Copy all errors into doc/ref/chap-errors.md
23 - This helps understand the language
24
25- Update doc/htm8.md
26- list of Algorithms:
27 - lex just the top level
28 - lex both levels
29 - and match tags - this is the level for value.Htm8Frag?
30 - convert to XML!
31 - lazy selection by tag, or attr (id= and class=)
32 - lazy selection by CSS selector expression
33 - convert to DOMTree
34 - sed-like replacement of DOM Tree or element
35 - untrusted HTML filter, e.g. like StackOverflow / Reddit
36 - this is Safe HTM8
37 - should have a zero alloc way to support this, with good errors?
38 - I think most of them silently strip data
39"""
40
41import re
42
43from typing import Dict, List, Tuple, Optional, IO, Iterator, Any
44
45from _devbuild.gen.htm8_asdl import (h8_id, h8_id_t, h8_tag_id, h8_tag_id_t,
46 h8_tag_id_str, attr_name, attr_name_t,
47 attr_value, attr_value_t, attr_value_id)
48from doctools.util import log
49
50
51class LexError(Exception):
52 """
53 Examples of lex errors:
54
55 - h8_id.Invalid, like <> or &&
56 - Unclosed <!-- <? <![CDATA[ <script> <style>
57 """
58
59 def __init__(self, s, start_pos):
60 # type: (str, int) -> None
61 self.s = s
62 self.start_pos = start_pos
63
64 def __str__(self):
65 # type: () -> str
66 return '(LexError %r)' % (self.s[self.start_pos:self.start_pos + 20])
67
68
69def _FindLineNum(s, error_pos):
70 # type: (str, int) -> int
71 current_pos = 0
72 line_num = 1
73 while True:
74 newline_pos = s.find('\n', current_pos)
75 #log('current = %d, N %d, line %d', current_pos, newline_pos, line_num)
76
77 if newline_pos == -1: # this is the last line
78 return line_num
79 if newline_pos >= error_pos:
80 return line_num
81 line_num += 1
82 current_pos = newline_pos + 1
83
84
85class ParseError(Exception):
86 """
87 Examples of parse errors
88
89 - unbalanced tag structure
90 - ul_table.py errors
91 """
92
93 def __init__(self, msg, s=None, start_pos=-1):
94 # type: (str, Optional[str], int) -> None
95 self.msg = msg
96 self.s = s
97 self.start_pos = start_pos
98
99 def __str__(self):
100 # type: () -> str
101 if self.s is not None:
102 assert self.start_pos != -1, self.start_pos
103 snippet = (self.s[self.start_pos:self.start_pos + 20])
104
105 line_num = _FindLineNum(self.s, self.start_pos)
106 else:
107 snippet = ''
108 line_num = -1
109 msg = 'line %d: %r %r' % (line_num, self.msg, snippet)
110 return msg
111
112
113class Output(object):
114 """Takes an underlying input buffer and an output file. Maintains a
115 position in the input buffer.
116
117 Print FROM the input or print new text to the output.
118 """
119
120 def __init__(self, s, f, left_pos=0, right_pos=-1):
121 # type: (str, IO[str], int, int) -> None
122 self.s = s
123 self.f = f
124 self.pos = left_pos
125 self.right_pos = len(s) if right_pos == -1 else right_pos
126
127 def SkipTo(self, pos):
128 # type: (int) -> None
129 """Skip to a position."""
130 self.pos = pos
131
132 def PrintUntil(self, pos):
133 # type: (int) -> None
134 """Print until a position."""
135 piece = self.s[self.pos:pos]
136 self.f.write(piece)
137 self.pos = pos
138
139 def PrintTheRest(self):
140 # type: () -> None
141 """Print until the end of the string."""
142 self.PrintUntil(self.right_pos)
143
144 def Print(self, s):
145 # type: (str) -> None
146 """Print text to the underlying buffer."""
147 self.f.write(s)
148
149
150def MakeLexer(rules):
151 return [(re.compile(pat, re.VERBOSE), i) for (pat, i) in rules]
152
153
154#
155# Eggex
156#
157# Tag = / ~['>']+ /
158
159# Is this valid? A single character?
160# Tag = / ~'>'* /
161
162# Maybe better: / [NOT '>']+/
163# capital letters not allowed there?
164#
165# But then this is confusing:
166# / [NOT ~digit]+/
167#
168# / [NOT digit] / is [^\d]
169# / ~digit / is \D
170#
171# Or maybe:
172#
173# / [~ digit]+ /
174# / [~ '>']+ /
175# / [NOT '>']+ /
176
177# End = / '</' Tag '>' /
178# StartEnd = / '<' Tag '/>' /
179# Start = / '<' Tag '>' /
180#
181# EntityRef = / '&' dot{* N} ';' /
182
183# Tag name, or attribute name
184# colon is used in XML
185
186# https://www.w3.org/TR/xml/#NT-Name
187# Hm there is a lot of unicode stuff. We are simplifying parsing
188
189_NAME = r'[a-zA-Z][a-zA-Z0-9:_\-]*' # must start with letter
190
191CHAR_LEX = [
192 # Characters
193 # https://www.w3.org/TR/xml/#sec-references
194 (r'&\# [0-9]+ ;', h8_id.DecChar),
195 (r'&\# x[0-9a-fA-F]+ ;', h8_id.HexChar),
196 (r'& %s ;' % _NAME, h8_id.CharEntity),
197 # Allow unquoted, and quoted
198 (r'&', h8_id.BadAmpersand),
199]
200
201HTM8_LEX = CHAR_LEX + [
202 # TODO: CommentBegin, ProcessingBegin, CDataBegin could have an additional
203 # action associated with them? The ending substring
204 (r'<!--', h8_id.CommentBegin),
205
206 # Processing instruction are used for the XML header:
207 # <?xml version="1.0" encoding="UTF-8"?>
208 # They are technically XML-only, but in HTML5, they are another kind of
209 # comment:
210 #
211 # https://developer.mozilla.org/en-US/docs/Web/API/ProcessingInstruction
212 #
213 (r'<\?', h8_id.ProcessingBegin),
214 # Not necessary in HTML5, but occurs in XML
215 (r'<!\[CDATA\[', h8_id.CDataBegin), # <![CDATA[
216
217 # Markup declarations
218 # - In HTML5, there is only <!DOCTYPE html>
219 # - XML has 4 more declarations: <!ELEMENT ...> ATTLIST ENTITY NOTATION
220 # - these seem to be part of DTD
221 # - it's useful to skip these, and be able to parse the rest of the document
222 # - Note: < is allowed?
223 (r'<! [^>\x00]+ >', h8_id.Decl),
224
225 # Tags
226 # Notes:
227 # - We look for a valid tag name, but we don't validate attributes.
228 # That's done in the tag lexer.
229 # - We don't allow leading whitespace
230 (r'</ (%s) >' % _NAME, h8_id.EndTag),
231 # self-closing <br/> comes before StartTag
232 # could/should these be collapsed into one rule?
233 (r'< (%s) [^>\x00]* />' % _NAME, h8_id.StartEndTag), # end </a>
234 (r'< (%s) [^>\x00]* >' % _NAME, h8_id.StartTag), # start <a>
235
236 # HTML5 allows unescaped > in raw data, but < is not allowed.
237 # https://stackoverflow.com/questions/10462348/right-angle-bracket-in-html
238 #
239 # - My early blog has THREE errors when disallowing >
240 # - So do some .wwz files
241 (r'[^&<>\x00]+', h8_id.RawData),
242 (r'>', h8_id.BadGreaterThan),
243 # NUL is the end, an accomodation for re2c. Like we do in frontend/match.
244 (r'\x00', h8_id.EndOfStream),
245 # < is an error
246 (r'.', h8_id.Invalid),
247]
248
249# Old notes:
250#
251# Non-greedy matches are regular and can be matched in linear time
252# with RE2.
253#
254# https://news.ycombinator.com/item?id=27099798
255#
256
257# This person tried to do it with a regex:
258#
259# https://skeptric.com/html-comment-regexp/index.html
260
261# . is any char except newline
262# https://re2c.org/manual/manual_c.html
263
264# Discarded options
265#(r'<!-- .*? -->', h8_id.Comment),
266
267# Hack from Claude: \s\S instead of re.DOTALL. I don't like this
268#(r'<!-- [\s\S]*? -->', h8_id.Comment),
269#(r'<!-- (?:.|[\n])*? -->', h8_id.Comment),
270
271HTM8_LEX_COMPILED = MakeLexer(HTM8_LEX)
272
273
274class Lexer(object):
275
276 def __init__(self, s, left_pos=0, right_pos=-1, no_special_tags=False):
277 # type: (str, int, int, bool) -> None
278 self.s = s
279 self.pos = left_pos
280 self.right_pos = len(s) if right_pos == -1 else right_pos
281 self.no_special_tags = no_special_tags
282
283 # string -> compiled regex pattern object
284 self.cache = {} # type: Dict[str, Any]
285
286 # either </script> or </style> - we search until we see that
287 self.search_state = None # type: Optional[str]
288
289 # Position of tag name, if applicable
290 # - Set after you get a StartTag, EndTag, or StartEndTag
291 # - Unset on other tags
292 self.tag_pos_left = -1
293 self.tag_pos_right = -1
294
295 def _Read(self):
296 # type: () -> Tuple[h8_id_t, int]
297 if self.pos == self.right_pos:
298 return h8_id.EndOfStream, self.pos
299
300 assert self.pos < self.right_pos, self.pos
301
302 if self.search_state is not None and not self.no_special_tags:
303 # TODO: case-insensitive search for </SCRIPT> <SCRipt> ?
304 #
305 # Another strategy: enter a mode where we find ONLY the end tag
306 # regex, and any data that's not <, and then check the canonical
307 # tag name for 'script' or 'style'.
308 pos = self.s.find(self.search_state, self.pos)
309 if pos == -1:
310 # unterminated <script> or <style>
311 raise LexError(self.s, self.pos)
312 self.search_state = None
313 # beginning
314 return h8_id.HtmlCData, pos
315
316 # Find the first match.
317 # Note: frontend/match.py uses _LongestMatch(), which is different!
318 # TODO: reconcile them. This lexer should be expressible in re2c.
319
320 for pat, tok_id in HTM8_LEX_COMPILED:
321 m = pat.match(self.s, self.pos)
322 if m:
323 if tok_id in (h8_id.StartTag, h8_id.EndTag, h8_id.StartEndTag):
324 self.tag_pos_left = m.start(1)
325 self.tag_pos_right = m.end(1)
326 else:
327 # Reset state
328 self.tag_pos_left = -1
329 self.tag_pos_right = -1
330
331 if tok_id == h8_id.CommentBegin:
332 pos = self.s.find('-->', self.pos)
333 if pos == -1:
334 # unterminated <!--
335 raise LexError(self.s, self.pos)
336 return h8_id.Comment, pos + 3 # -->
337
338 if tok_id == h8_id.ProcessingBegin:
339 pos = self.s.find('?>', self.pos)
340 if pos == -1:
341 # unterminated <?
342 raise LexError(self.s, self.pos)
343 return h8_id.Processing, pos + 2 # ?>
344
345 if tok_id == h8_id.CDataBegin:
346 pos = self.s.find(']]>', self.pos)
347 if pos == -1:
348 # unterminated <![CDATA[
349 raise LexError(self.s, self.pos)
350 return h8_id.CData, pos + 3 # ]]>
351
352 if tok_id == h8_id.StartTag:
353 # TODO: reduce allocations
354 if (self.TagNameEquals('script') or
355 self.TagNameEquals('style')):
356 # <SCRipt a=b> -> </SCRipt>
357 self.search_state = '</' + self._LiteralTagName() + '>'
358
359 return tok_id, m.end()
360 else:
361 raise AssertionError('h8_id.Invalid rule should have matched')
362
363 def TagNamePos(self):
364 """The right position of the tag pos"""
365 assert self.tag_pos_right != -1, self.tag_pos_right
366 return self.tag_pos_right
367
368 def TagNameEquals(self, expected):
369 # type: (str) -> bool
370 assert self.tag_pos_left != -1, self.tag_pos_left
371 assert self.tag_pos_right != -1, self.tag_pos_right
372
373 # TODO: In C++, this does not need an allocation. Can we test
374 # directly?
375 return expected == self.CanonicalTagName()
376
377 def _LiteralTagName(self):
378 # type: () -> str
379 assert self.tag_pos_left != -1, self.tag_pos_left
380 assert self.tag_pos_right != -1, self.tag_pos_right
381
382 return self.s[self.tag_pos_left:self.tag_pos_right]
383
384 def CanonicalTagName(self):
385 # type: () -> str
386 tag_name = self._LiteralTagName()
387 # Most tags are already lower case, so avoid allocation with this conditional
388 # TODO: this could go in the mycpp runtime?
389 if tag_name.islower():
390 return tag_name
391 else:
392 return tag_name.lower()
393
394 def Read(self):
395 # type: () -> Tuple[h8_id_t, int]
396 tok_id, end_pos = self._Read()
397 self.pos = end_pos # advance
398 return tok_id, end_pos
399
400 def LookAhead(self, regex):
401 # type: (str) -> bool
402 """
403 Currently used for ul_table.py. But taking a dynamic regex string is
404 not the right interface.
405 """
406 # Cache the regex compilation. This could also be LookAheadFor(THEAD)
407 # or something.
408 pat = self.cache.get(regex)
409 if pat is None:
410 pat = re.compile(regex)
411 self.cache[regex] = pat
412
413 m = pat.match(self.s, self.pos)
414 return m is not None
415
416
417A_NAME_LEX = [
418 # Leading whitespace is required, to separate attributes.
419 #
420 # If the = is not present, then we set the lexer in a state for
421 # attr_value.Missing.
422 (r'\s+ (%s) \s* (=)?' % _NAME, attr_name.Ok),
423 # unexpected EOF
424
425 # The closing > or /> is treated as end of stream, and it's not an error.
426 (r'\s* /? >', attr_name.Done),
427
428 # NUL should not be possible, because the top-level
429
430 # e.g. < is an error
431 (r'.', attr_name.Invalid),
432]
433
434A_NAME_LEX_COMPILED = MakeLexer(A_NAME_LEX)
435
436# Here we just loop on regular tokens
437#
438# Examples:
439# <a href = unquoted&amp;foo >
440# <a href = unquoted&foo > # BadAmpersand is allowed I guess
441# <a href ="unquoted&foo" > # double quoted
442# <a href ='unquoted&foo' > # single quoted
443# <a href = what"foo" > # HTML5 allows this, but we could disallow it if
444# it's not common. It opens up the j"" and $"" extensions
445# <a href = what'foo' > # ditto
446#
447# Problem: <a href=foo/> - this is hard to recognize
448# Because is the unquoted value "foo/" or "foo" ?
449
450# Be very lenient - just no whitespace or special HTML chars
451# I don't think this is more lenient than HTML5, though we should check.
452#
453# Bug fix: Also disallow /
454
455_UNQUOTED_VALUE = r'''[^ \t\r\n<>&/"'\x00]*'''
456
457A_VALUE_LEX = CHAR_LEX + [
458 (r'"', attr_value_id.DoubleQuote),
459 (r"'", attr_value_id.SingleQuote),
460 (_UNQUOTED_VALUE, attr_value_id.UnquotedVal),
461
462 #(r'[ \r\n\t]', h8_id.Whitespace), # terminates unquoted values
463 #(r'[^ \r\n\t&>\x00]', h8_id.RawData),
464 #(r'[>\x00]', h8_id.EndOfStream),
465 # e.g. < is an error
466 (r'.', attr_value_id.Invalid),
467]
468
469A_VALUE_LEX_COMPILED = MakeLexer(A_VALUE_LEX)
470
471
472class AttrLexer(object):
473 """
474 We can also invert this
475
476 Unquoted (List[h8_id] tok_ids, List[int] end_pos)
477
478 It would be nice to have a special case for the singleton, since that is
479 very common.
480
481 Simple (int tag_name_start, int tag_name_end, int attr_value_tag,
482 int value_start, int value_end)
483 This would cover many cases
484
485 The other option is to create many different events, and have AttrValueLexer
486 But I think that is annoying and overly detailed.
487
488 Operations:
489 - GetAttrRaw('foo')
490 - AllAttrsRaw()
491 - AllAttrsRawSlice()
492
493 class= query - well we can do this with Space tokens I think - we should
494 have an optimization
495 id= query - ditto, we should just have a predicate
496
497 Zero allocs:
498 tag query - TagNameEquals()
499
500 So I guess we have to write a HasClass('foo') and IdEquals('bar') on top of
501 this. Yes.
502
503 tag_lx.Reset2(...) # we should pass it the tag_name_end position
504 tag_lx.Read() -> bool # success or fail? Or Attr or Invalid
505 .AttrNameEquals('foo') -> bool # id and class query
506 .GetAttrName() -> str # for getting them all
507 .GetRawValue() -> Tuple[h8_tag_id, start, end] # just beginning and end
508 .GetValueTokens() -> Tuple[h8_tag_id, TokenList]
509 .TokenList = Tuple[List[h8_id], List[int end_pos]
510
511 You could also have
512
513 tag_lx.GetValueTokenId() -> Tuple[h8_id, end_pos]
514
515 And then you read it until it's " or ' or space ? We probably won't have
516 that use case to start.
517 """
518
519 def __init__(self, s):
520 # type: (str) -> None
521 self.s = s
522 self.tag_name_pos = -1 # Invalid
523 self.tag_end_pos = -1
524 self.pos = -1
525
526 self.name_start = -1
527 self.name_end = -1
528 self.next_value_is_missing = False
529
530 def Init(self, tag_name_pos, end_pos):
531 # type: (int, int) -> None
532 """Initialize so we can read names and values.
533
534 Example:
535 'x <a y>' # tag_name_pos=4, end_pos=6
536 'x <a>' # tag_name_pos=4, end_pos=4
537
538 The Reset() method is used to reuse instances of the AttrLexer object.
539 """
540 assert tag_name_pos >= 0, tag_name_pos
541 assert end_pos >= 0, end_pos
542
543 log('TAG NAME POS %d', tag_name_pos)
544
545 self.tag_name_pos = tag_name_pos
546 self.end_pos = end_pos
547
548 self.pos = tag_name_pos
549
550 def ReadName(self):
551 # type: () -> Tuple[attr_name_t, int, int]
552 """Reads the attribute name
553
554 EOF case:
555 <a>
556 <a >
557
558 Error case:
559 <a !>
560 <a foo=bar !>
561 """
562 for pat, a in A_NAME_LEX_COMPILED:
563 m = pat.match(self.s, self.pos)
564 if m:
565 if a == attr_name.Ok:
566 #log('%r', m.groups())
567 self.name_start = m.start(1)
568 self.name_end = m.end(1)
569 # Set state based on =
570 if m.group(2) is None:
571 self.next_value_is_missing = True
572 return attr_name.Ok, self.name_start, self.name_end
573 else:
574 # Reset state - e.g. you must call AttrNameEquals
575 self.name_start = -1
576 self.name_end = -1
577 self.next_value_is_missing = False
578
579 if a == attr_name.Invalid:
580 return attr_name.Invalid, -1, -1
581 if a == attr_name.Done:
582 return attr_name.Done, -1, -1
583 else:
584 raise AssertionError('h8_id.Invalid rule should have matched')
585
586 def _CanonicalAttrName(self):
587 # type: () -> str
588 assert self.name_start >= 0, self.name_start
589 assert self.name_end >= 0, self.name_end
590
591 attr_name = self.s[self.name_start:self.name_end]
592 if attr_name.islower():
593 return attr_name
594 else:
595 return attr_name.lower()
596
597 def AttrNameEquals(self, expected):
598 # type: (str) -> bool
599 """
600 TODO: Must call this after ReadName() ?
601 Because that can FAIL.
602 """
603 return expected == self._CanonicalAttrName()
604
605 def ReadRawValue(self):
606 # type: () -> Tuple[attr_value_t, int, int]
607 """Read the attribute value.
608
609 In general, it is escaped or "raw"
610
611 Note: Assuming ReadName() returned a value, this should NOT fail.
612 """
613 # NOTE: if = is not found, set state
614 _ = attr_value
615
616 pass
617
618 def SkipValue(self):
619 # type: () -> None
620 # Just ignore it and return
621 self.ReadRawValue()
622
623 def ReadValueAndDecode(self):
624 # type: () -> str
625 """Read the attribute vlaue
626 """
627 # TODO: tokenize it
628 pass
629
630
631# Tag names:
632# Match <a or </a
633# Match <h2, but not <2h
634#
635# HTML 5 doesn't restrict tag names at all
636# https://html.spec.whatwg.org/#toc-syntax
637#
638# XML allows : - .
639# https://www.w3.org/TR/xml/#NT-NameChar
640
641# Namespaces for MathML, SVG
642# XLink, XML, XMLNS
643#
644# https://infra.spec.whatwg.org/#namespaces
645#
646# Allow - for td-attrs
647
648# TODO: we don't need to capture the tag name here? That's done at the top
649# level
650_TAG_RE = re.compile(r'/? \s* (%s)' % _NAME, re.VERBOSE)
651
652_TAG_LAST_RE = re.compile(r'\s* /? >', re.VERBOSE)
653
654# To match href="foo"
655# Note: in HTML5 and XML, single quoted attributes are also valid
656
657# <button disabled> is standard usage
658
659# NOTE: This used to allow whitespace around =
660# <a foo = "bar"> makes sense in XML
661# But then you also have
662# <a foo= bar> - which is TWO attributes, in HTML5
663# So the space is problematic
664
665_ATTR_RE = re.compile(
666 r'''
667\s+ # Leading whitespace is required
668(%s) # Attribute name
669(?: # Optional attribute value
670 \s* = \s* # Spaces allowed around =
671 (?:
672 " ([^>"\x00]*) " # double quoted value
673 | ' ([^>'\x00]*) ' # single quoted value
674 | (%s) # Attribute value
675 )
676)?
677''' % (_NAME, _UNQUOTED_VALUE), re.VERBOSE)
678
679
680class TagLexer(object):
681 """
682 Given a tag like <a href="..."> or <link type="..." />, the TagLexer
683 provides a few operations:
684
685 - What is the tag?
686 - Iterate through the attributes, giving (name, value_start_pos, value_end_pos)
687 """
688
689 def __init__(self, s):
690 # type: (str) -> None
691 self.s = s
692 self.start_pos = -1 # Invalid
693 self.end_pos = -1
694
695 def Reset(self, start_pos, end_pos):
696 # type: (int, int) -> None
697 """Reuse instances of this object."""
698 assert start_pos >= 0, start_pos
699 assert end_pos >= 0, end_pos
700
701 self.start_pos = start_pos
702 self.end_pos = end_pos
703
704 def WholeTagString(self):
705 # type: () -> str
706 """Return the entire tag string, e.g. <a href='foo'>"""
707 return self.s[self.start_pos:self.end_pos]
708
709 def GetTagName(self):
710 # type: () -> str
711 # First event
712 tok_id, start, end = next(self.Tokens())
713 return self.s[start:end]
714
715 def GetSpanForAttrValue(self, attr_name):
716 # type: (str) -> Tuple[int, int]
717 """
718 Used by oils_doc.py, for href shortcuts
719 """
720 # Algorithm: search for QuotedValue or UnquotedValue after AttrName
721 # TODO: Could also cache these
722
723 events = self.Tokens()
724 val = (-1, -1)
725 try:
726 while True:
727 tok_id, start, end = next(events)
728 if tok_id == h8_tag_id.AttrName:
729 name = self.s[start:end]
730 if name == attr_name:
731 # The value should come next
732 tok_id, start, end = next(events)
733 assert tok_id in (
734 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
735 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
736 val = start, end
737 break
738
739 except StopIteration:
740 pass
741 return val
742
743 def GetAttrRaw(self, attr_name):
744 # type: (str) -> Optional[str]
745 """
746 Return the value, which may be UNESCAPED.
747 """
748 start, end = self.GetSpanForAttrValue(attr_name)
749 if start == -1:
750 return None
751 return self.s[start:end]
752
753 def AllAttrsRawSlice(self):
754 # type: () -> List[Tuple[str, int, int]]
755 """
756 Get a list of pairs [('class', 3, 5), ('href', 9, 12)]
757 """
758 slices = []
759 events = self.Tokens()
760 try:
761 while True:
762 tok_id, start, end = next(events)
763 if tok_id == h8_tag_id.AttrName:
764 name = self.s[start:end]
765
766 # The value should come next
767 tok_id, start, end = next(events)
768 assert tok_id in (
769 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
770 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
771 # Note: quoted values may have &amp;
772 # We would need ANOTHER lexer to unescape them, but we
773 # don't need that for ul-table
774 slices.append((name, start, end))
775 except StopIteration:
776 pass
777 return slices
778
779 def AllAttrsRaw(self):
780 # type: () -> List[Tuple[str, str]]
781 """
782 Get a list of pairs [('class', 'foo'), ('href', '?foo=1&amp;bar=2')]
783
784 The quoted values may be escaped. We would need another lexer to
785 unescape them.
786 """
787 slices = self.AllAttrsRawSlice()
788 pairs = []
789 for name, start, end in slices:
790 pairs.append((name, self.s[start:end]))
791 return pairs
792
793 def Tokens(self):
794 # type: () -> Iterator[Tuple[h8_tag_id_t, int, int]]
795 """
796 Yields a sequence of tokens: Tag (AttrName AttrValue?)*
797
798 Where each Token is (Type, start_pos, end_pos)
799
800 Note that start and end are NOT redundant! We skip over some unwanted
801 characters.
802 """
803 m = _TAG_RE.match(self.s, self.start_pos + 1)
804 if not m:
805 raise RuntimeError("Couldn't find HTML tag in %r" %
806 self.WholeTagString())
807 yield h8_tag_id.TagName, m.start(1), m.end(1)
808
809 pos = m.end(0)
810 #log('POS %d', pos)
811
812 while True:
813 # don't search past the end
814 m = _ATTR_RE.match(self.s, pos, self.end_pos)
815 if not m:
816 #log('BREAK pos %d', pos)
817 break
818 #log('AttrName %r', m.group(1))
819
820 yield h8_tag_id.AttrName, m.start(1), m.end(1)
821
822 #log('m.groups() %r', m.groups())
823 if m.group(2) is not None:
824 # double quoted
825 yield h8_tag_id.QuotedValue, m.start(2), m.end(2)
826 elif m.group(3) is not None:
827 # single quoted - TODO: could have different token types
828 yield h8_tag_id.QuotedValue, m.start(3), m.end(3)
829 elif m.group(4) is not None:
830 yield h8_tag_id.UnquotedValue, m.start(4), m.end(4)
831 else:
832 # <button disabled>
833 end = m.end(0)
834 yield h8_tag_id.MissingValue, end, end
835
836 # Skip past the "
837 pos = m.end(0)
838
839 #log('TOK %r', self.s)
840
841 m = _TAG_LAST_RE.match(self.s, pos)
842 #log('_TAG_LAST_RE match %r', self.s[pos:])
843 if not m:
844 # Extra data at end of tag. TODO: add messages for all these.
845 raise LexError(self.s, pos)
846
847
848# This is similar but not identical to
849# " ([^>"\x00]*) " # double quoted value
850# | ' ([^>'\x00]*) ' # single quoted value
851#
852# Note: for unquoted values, & isn't allowed, and thus &amp; and &#99; and
853# &#x99; are not allowed. We could relax that?
854ATTR_VALUE_LEX = CHAR_LEX + [
855 (r'[^>&\x00]+', h8_id.RawData),
856 (r'.', h8_id.Invalid),
857]
858
859ATTR_VALUE_LEX_COMPILED = MakeLexer(ATTR_VALUE_LEX)
860
861
862class AttrValueLexer(object):
863 """
864 <a href="foo=99&amp;bar">
865 <a href='foo=99&amp;bar'>
866 <a href=unquoted>
867 """
868
869 def __init__(self, s):
870 # type: (str) -> None
871 self.s = s
872 self.start_pos = -1 # Invalid
873 self.end_pos = -1
874
875 def Reset(self, start_pos, end_pos):
876 # type: (int, int) -> None
877 """Reuse instances of this object."""
878 assert start_pos >= 0, start_pos
879 assert end_pos >= 0, end_pos
880
881 self.start_pos = start_pos
882 self.end_pos = end_pos
883
884 def NumTokens(self):
885 # type: () -> int
886 num_tokens = 0
887 pos = self.start_pos
888 for tok_id, end_pos in self.Tokens():
889 if tok_id == h8_id.Invalid:
890 raise LexError(self.s, pos)
891 pos = end_pos
892 #log('pos %d', pos)
893 num_tokens += 1
894 return num_tokens
895
896 def Tokens(self):
897 # type: () -> Iterator[Tuple[h8_id_t, int]]
898 pos = self.start_pos
899 while pos < self.end_pos:
900 # Find the first match, like above.
901 # Note: frontend/match.py uses _LongestMatch(), which is different!
902 # TODO: reconcile them. This lexer should be expressible in re2c.
903 for pat, tok_id in ATTR_VALUE_LEX_COMPILED:
904 m = pat.match(self.s, pos)
905 if m:
906 if 0:
907 tok_str = m.group(0)
908 log('token = %r', tok_str)
909
910 end_pos = m.end(0)
911 yield tok_id, end_pos
912 pos = end_pos
913 break
914 else:
915 raise AssertionError('h8_id.Invalid rule should have matched')