OILS / data_lang / htm8.py View on Github | oils.pub

962 lines, 404 significant
1"""data_lang/htm8.py
2
3TODO
4
5API:
6- Get rid of AttrValueLexer - this should be in the TagLexer
7 - this also means that unquoted values can be more similar
8 - We can use a single lexer mode for everything inside <>
9 - the SPACE is the only difference
10- Deprecate tag_lexer.GetTagName() in favor of lx.CanonicalTagName() or
11 _LiteralTagName()
12- UTF-8 check, like JSON8
13- re2c
14 - port lexer, which will fix static typing issues
15 - the abstraction needs to support submatch?
16 - for finding the end of a tag, etc.?
17
18- LexError and ParseError need details
19 - harmonize with data_lang/j8.py, which uses error.Decode(msg, ...,
20 cur_line_num)
21
22- Copy all errors into doc/ref/chap-errors.md
23 - This helps understand the language
24
25- Update doc/htm8.md
26- list of Algorithms:
27 - lex just the top level
28 - lex both levels
29 - and match tags - this is the level for value.Htm8Frag?
30 - convert to XML!
31 - lazy selection by tag, or attr (id= and class=)
32 - lazy selection by CSS selector expression
33 - convert to DOMTree
34 - sed-like replacement of DOM Tree or element
35 - untrusted HTML filter, e.g. like StackOverflow / Reddit
36 - this is Safe HTM8
37 - should have a zero alloc way to support this, with good errors?
38 - I think most of them silently strip data
39"""
40
41import re
42
43from typing import Dict, List, Tuple, Optional, IO, Iterator, Any
44
45from _devbuild.gen.htm8_asdl import (h8_id, h8_id_t, h8_tag_id, h8_tag_id_t,
46 h8_tag_id_str, attr_name, attr_name_t,
47 attr_value_e, attr_value_t, h8_val_id)
48from doctools.util import log
49
50
51class LexError(Exception):
52 """
53 Examples of lex errors:
54
55 - h8_id.Invalid, like <> or &&
56 - Unclosed <!-- <? <![CDATA[ <script> <style>
57 """
58
59 def __init__(self, s, start_pos):
60 # type: (str, int) -> None
61 self.s = s
62 self.start_pos = start_pos
63
64 def __str__(self):
65 # type: () -> str
66 return '(LexError %r)' % (self.s[self.start_pos:self.start_pos + 20])
67
68
69def _FindLineNum(s, error_pos):
70 # type: (str, int) -> int
71 current_pos = 0
72 line_num = 1
73 while True:
74 newline_pos = s.find('\n', current_pos)
75 #log('current = %d, N %d, line %d', current_pos, newline_pos, line_num)
76
77 if newline_pos == -1: # this is the last line
78 return line_num
79 if newline_pos >= error_pos:
80 return line_num
81 line_num += 1
82 current_pos = newline_pos + 1
83
84
85class ParseError(Exception):
86 """
87 Examples of parse errors
88
89 - unbalanced tag structure
90 - ul_table.py errors
91 """
92
93 def __init__(self, msg, s=None, start_pos=-1):
94 # type: (str, Optional[str], int) -> None
95 self.msg = msg
96 self.s = s
97 self.start_pos = start_pos
98
99 def __str__(self):
100 # type: () -> str
101 if self.s is not None:
102 assert self.start_pos != -1, self.start_pos
103 snippet = (self.s[self.start_pos:self.start_pos + 20])
104
105 line_num = _FindLineNum(self.s, self.start_pos)
106 else:
107 snippet = ''
108 line_num = -1
109 msg = 'line %d: %r %r' % (line_num, self.msg, snippet)
110 return msg
111
112
113class Output(object):
114 """Takes an underlying input buffer and an output file. Maintains a
115 position in the input buffer.
116
117 Print FROM the input or print new text to the output.
118 """
119
120 def __init__(self, s, f, left_pos=0, right_pos=-1):
121 # type: (str, IO[str], int, int) -> None
122 self.s = s
123 self.f = f
124 self.pos = left_pos
125 self.right_pos = len(s) if right_pos == -1 else right_pos
126
127 def SkipTo(self, pos):
128 # type: (int) -> None
129 """Skip to a position."""
130 self.pos = pos
131
132 def PrintUntil(self, pos):
133 # type: (int) -> None
134 """Print until a position."""
135 piece = self.s[self.pos:pos]
136 self.f.write(piece)
137 self.pos = pos
138
139 def PrintTheRest(self):
140 # type: () -> None
141 """Print until the end of the string."""
142 self.PrintUntil(self.right_pos)
143
144 def Print(self, s):
145 # type: (str) -> None
146 """Print text to the underlying buffer."""
147 self.f.write(s)
148
149
150def MakeLexer(rules):
151 return [(re.compile(pat, re.VERBOSE), i) for (pat, i) in rules]
152
153
154#
155# Eggex
156#
157# Tag = / ~['>']+ /
158
159# Is this valid? A single character?
160# Tag = / ~'>'* /
161
162# Maybe better: / [NOT '>']+/
163# capital letters not allowed there?
164#
165# But then this is confusing:
166# / [NOT ~digit]+/
167#
168# / [NOT digit] / is [^\d]
169# / ~digit / is \D
170#
171# Or maybe:
172#
173# / [~ digit]+ /
174# / [~ '>']+ /
175# / [NOT '>']+ /
176
177# End = / '</' Tag '>' /
178# StartEnd = / '<' Tag '/>' /
179# Start = / '<' Tag '>' /
180#
181# EntityRef = / '&' dot{* N} ';' /
182
183# Tag name, or attribute name
184# colon is used in XML
185
186# https://www.w3.org/TR/xml/#NT-Name
187# Hm there is a lot of unicode stuff. We are simplifying parsing
188
189_NAME = r'[a-zA-Z][a-zA-Z0-9:_\-]*' # must start with letter
190
191CHAR_LEX = [
192 # Characters
193 # https://www.w3.org/TR/xml/#sec-references
194 (r'&\# [0-9]+ ;', h8_id.DecChar),
195 (r'&\# x[0-9a-fA-F]+ ;', h8_id.HexChar),
196 (r'& %s ;' % _NAME, h8_id.CharEntity),
197 # Allow unquoted, and quoted
198 (r'&', h8_id.BadAmpersand),
199]
200
201HTM8_LEX = CHAR_LEX + [
202 # TODO: CommentBegin, ProcessingBegin, CDataBegin could have an additional
203 # action associated with them? The ending substring
204 (r'<!--', h8_id.CommentBegin),
205
206 # Processing instruction are used for the XML header:
207 # <?xml version="1.0" encoding="UTF-8"?>
208 # They are technically XML-only, but in HTML5, they are another kind of
209 # comment:
210 #
211 # https://developer.mozilla.org/en-US/docs/Web/API/ProcessingInstruction
212 #
213 (r'<\?', h8_id.ProcessingBegin),
214 # Not necessary in HTML5, but occurs in XML
215 (r'<!\[CDATA\[', h8_id.CDataBegin), # <![CDATA[
216
217 # Markup declarations
218 # - In HTML5, there is only <!DOCTYPE html>
219 # - XML has 4 more declarations: <!ELEMENT ...> ATTLIST ENTITY NOTATION
220 # - these seem to be part of DTD
221 # - it's useful to skip these, and be able to parse the rest of the document
222 # - Note: < is allowed?
223 (r'<! [^>\x00]+ >', h8_id.Decl),
224
225 # Tags
226 # Notes:
227 # - We look for a valid tag name, but we don't validate attributes.
228 # That's done in the tag lexer.
229 # - We don't allow leading whitespace
230 (r'</ (%s) >' % _NAME, h8_id.EndTag),
231 # self-closing <br/> comes before StartTag
232 # could/should these be collapsed into one rule?
233 (r'< (%s) [^>\x00]* />' % _NAME, h8_id.StartEndTag), # end </a>
234 (r'< (%s) [^>\x00]* >' % _NAME, h8_id.StartTag), # start <a>
235
236 # HTML5 allows unescaped > in raw data, but < is not allowed.
237 # https://stackoverflow.com/questions/10462348/right-angle-bracket-in-html
238 #
239 # - My early blog has THREE errors when disallowing >
240 # - So do some .wwz files
241 (r'[^&<>\x00]+', h8_id.RawData),
242 (r'>', h8_id.BadGreaterThan),
243 # NUL is the end, an accomodation for re2c. Like we do in frontend/match.
244 (r'\x00', h8_id.EndOfStream),
245 # This includes < - it is not BadLessThan because it's NOT recoverable
246 (r'.', h8_id.Invalid),
247]
248
249# Old notes:
250#
251# Non-greedy matches are regular and can be matched in linear time
252# with RE2.
253#
254# https://news.ycombinator.com/item?id=27099798
255#
256
257# This person tried to do it with a regex:
258#
259# https://skeptric.com/html-comment-regexp/index.html
260
261# . is any char except newline
262# https://re2c.org/manual/manual_c.html
263
264# Discarded options
265#(r'<!-- .*? -->', h8_id.Comment),
266
267# Hack from Claude: \s\S instead of re.DOTALL. I don't like this
268#(r'<!-- [\s\S]*? -->', h8_id.Comment),
269#(r'<!-- (?:.|[\n])*? -->', h8_id.Comment),
270
271HTM8_LEX_COMPILED = MakeLexer(HTM8_LEX)
272
273
274class Lexer(object):
275
276 def __init__(self, s, left_pos=0, right_pos=-1, no_special_tags=False):
277 # type: (str, int, int, bool) -> None
278 self.s = s
279 self.pos = left_pos
280 self.right_pos = len(s) if right_pos == -1 else right_pos
281 self.no_special_tags = no_special_tags
282
283 # string -> compiled regex pattern object
284 self.cache = {} # type: Dict[str, Any]
285
286 # either </script> or </style> - we search until we see that
287 self.search_state = None # type: Optional[str]
288
289 # Position of tag name, if applicable
290 # - Set after you get a StartTag, EndTag, or StartEndTag
291 # - Unset on other tags
292 self.tag_pos_left = -1
293 self.tag_pos_right = -1
294
295 def _Read(self):
296 # type: () -> Tuple[h8_id_t, int]
297 if self.pos == self.right_pos:
298 return h8_id.EndOfStream, self.pos
299
300 assert self.pos < self.right_pos, self.pos
301
302 if self.search_state is not None and not self.no_special_tags:
303 # TODO: case-insensitive search for </SCRIPT> <SCRipt> ?
304 #
305 # Another strategy: enter a mode where we find ONLY the end tag
306 # regex, and any data that's not <, and then check the canonical
307 # tag name for 'script' or 'style'.
308 pos = self.s.find(self.search_state, self.pos)
309 if pos == -1:
310 # unterminated <script> or <style>
311 raise LexError(self.s, self.pos)
312 self.search_state = None
313 # beginning
314 return h8_id.HtmlCData, pos
315
316 # Find the first match.
317 # Note: frontend/match.py uses _LongestMatch(), which is different!
318 # TODO: reconcile them. This lexer should be expressible in re2c.
319
320 for pat, tok_id in HTM8_LEX_COMPILED:
321 m = pat.match(self.s, self.pos)
322 if m:
323 if tok_id in (h8_id.StartTag, h8_id.EndTag, h8_id.StartEndTag):
324 self.tag_pos_left = m.start(1)
325 self.tag_pos_right = m.end(1)
326 else:
327 # Reset state
328 self.tag_pos_left = -1
329 self.tag_pos_right = -1
330
331 if tok_id == h8_id.CommentBegin:
332 pos = self.s.find('-->', self.pos)
333 if pos == -1:
334 # unterminated <!--
335 raise LexError(self.s, self.pos)
336 return h8_id.Comment, pos + 3 # -->
337
338 if tok_id == h8_id.ProcessingBegin:
339 pos = self.s.find('?>', self.pos)
340 if pos == -1:
341 # unterminated <?
342 raise LexError(self.s, self.pos)
343 return h8_id.Processing, pos + 2 # ?>
344
345 if tok_id == h8_id.CDataBegin:
346 pos = self.s.find(']]>', self.pos)
347 if pos == -1:
348 # unterminated <![CDATA[
349 raise LexError(self.s, self.pos)
350 return h8_id.CData, pos + 3 # ]]>
351
352 if tok_id == h8_id.StartTag:
353 # TODO: reduce allocations
354 if (self.TagNameEquals('script') or
355 self.TagNameEquals('style')):
356 # <SCRipt a=b> -> </SCRipt>
357 self.search_state = '</' + self._LiteralTagName() + '>'
358
359 return tok_id, m.end()
360 else:
361 raise AssertionError('h8_id.Invalid rule should have matched')
362
363 def TagNamePos(self):
364 """The right position of the tag pos"""
365 assert self.tag_pos_right != -1, self.tag_pos_right
366 return self.tag_pos_right
367
368 def TagNameEquals(self, expected):
369 # type: (str) -> bool
370 assert self.tag_pos_left != -1, self.tag_pos_left
371 assert self.tag_pos_right != -1, self.tag_pos_right
372
373 # TODO: In C++, this does not need an allocation. Can we test
374 # directly?
375 return expected == self.CanonicalTagName()
376
377 def _LiteralTagName(self):
378 # type: () -> str
379 assert self.tag_pos_left != -1, self.tag_pos_left
380 assert self.tag_pos_right != -1, self.tag_pos_right
381
382 return self.s[self.tag_pos_left:self.tag_pos_right]
383
384 def CanonicalTagName(self):
385 # type: () -> str
386 tag_name = self._LiteralTagName()
387 # Most tags are already lower case, so avoid allocation with this conditional
388 # TODO: this could go in the mycpp runtime?
389 if tag_name.islower():
390 return tag_name
391 else:
392 return tag_name.lower()
393
394 def Read(self):
395 # type: () -> Tuple[h8_id_t, int]
396 tok_id, end_pos = self._Read()
397 self.pos = end_pos # advance
398 return tok_id, end_pos
399
400 def LookAhead(self, regex):
401 # type: (str) -> bool
402 """
403 Currently used for ul_table.py. But taking a dynamic regex string is
404 not the right interface.
405 """
406 # Cache the regex compilation. This could also be LookAheadFor(THEAD)
407 # or something.
408 pat = self.cache.get(regex)
409 if pat is None:
410 pat = re.compile(regex)
411 self.cache[regex] = pat
412
413 m = pat.match(self.s, self.pos)
414 return m is not None
415
416
417A_NAME_LEX = [
418 # Leading whitespace is required, to separate attributes.
419 #
420 # If the = is not present, then we set the lexer in a state for
421 # attr_value_e.Missing.
422 (r'\s+ (%s) \s* (=)? \s*' % _NAME, attr_name.Ok),
423 # unexpected EOF
424
425 # The closing > or /> is treated as end of stream, and it's not an error.
426 (r'\s* /? >', attr_name.Done),
427
428 # NUL should not be possible, because the top-level
429
430 # This includes < - it is not BadLessThan because it's NOT recoverable
431 (r'.', attr_name.Invalid),
432]
433
434A_NAME_LEX_COMPILED = MakeLexer(A_NAME_LEX)
435
436# Here we just loop on regular tokens
437#
438# Examples:
439# <a href = unquoted&amp;foo >
440# <a href = unquoted&foo > # BadAmpersand is allowed I guess
441# <a href ="unquoted&foo" > # double quoted
442# <a href ='unquoted&foo' > # single quoted
443# <a href = what"foo" > # HTML5 allows this, but we could disallow it if
444# it's not common. It opens up the j"" and $"" extensions
445# <a href = what'foo' > # ditto
446#
447# Problem: <a href=foo/> - this is hard to recognize
448# Because is the unquoted value "foo/" or "foo" ?
449
450# Be very lenient - just no whitespace or special HTML chars
451# I don't think this is more lenient than HTML5, though we should check.
452#
453# Bug fix: Also disallow /
454
455# TODO: get rid of OLD copy
456_UNQUOTED_VALUE_OLD = r'''[^ \t\r\n<>&/"'\x00]*'''
457_UNQUOTED_VALUE = r'''[^ \t\r\n<>&/"'\x00]+'''
458
459# Restrictive definition, similar to _NAME
460# I was trying to capture #ble.sh and so forth
461# I also had unquoted //github.com, etc.
462
463# _UNQUOTED_VALUE = r'''[a-zA-Z0-9:_\-]+'''
464#
465# For now, I guess we live with <a href=?foo/>
466
467A_VALUE_LEX = CHAR_LEX + [
468 (r'"', h8_val_id.DoubleQuote),
469 (r"'", h8_val_id.SingleQuote),
470 (_UNQUOTED_VALUE, h8_val_id.UnquotedVal),
471
472 #(r'[ \r\n\t]', h8_id.Whitespace), # terminates unquoted values
473 #(r'[^ \r\n\t&>\x00]', h8_id.RawData),
474 #(r'[>\x00]', h8_id.EndOfStream),
475 # e.g. < is an error
476 (r'.', h8_val_id.NoMatch),
477]
478
479A_VALUE_LEX_COMPILED = MakeLexer(A_VALUE_LEX)
480
481
482class AttrLexer(object):
483 """
484 Typical usage:
485
486 while True:
487 n, start_pos, end_pos = attr_lx.ReadName()
488 if n == attr_name.Ok:
489 if attr_lx.AttrNameEquals('div'):
490 print('div')
491
492 # TODO: also pass Optional[List[]] out_tokens?
493 v, start_pos, end_pos = attr_lx.ReadRawValue()
494 """
495
496 def __init__(self, s):
497 # type: (str) -> None
498 self.s = s
499 self.tag_name_pos = -1 # Invalid
500 self.tag_end_pos = -1
501 self.pos = -1
502
503 self.name_start = -1
504 self.name_end = -1
505 self.next_value_is_missing = False
506
507 def Init(self, tag_name_pos, end_pos):
508 # type: (int, int) -> None
509 """Initialize so we can read names and values.
510
511 Example:
512 'x <a y>' # tag_name_pos=4, end_pos=6
513 'x <a>' # tag_name_pos=4, end_pos=4
514
515 The Reset() method is used to reuse instances of the AttrLexer object.
516 """
517 assert tag_name_pos >= 0, tag_name_pos
518 assert end_pos >= 0, end_pos
519
520 log('TAG NAME POS %d', tag_name_pos)
521
522 self.tag_name_pos = tag_name_pos
523 self.end_pos = end_pos
524
525 self.pos = tag_name_pos
526
527 def ReadName(self):
528 # type: () -> Tuple[attr_name_t, int, int]
529 """Reads the attribute name
530
531 EOF case:
532 <a>
533 <a >
534
535 Error case:
536 <a !>
537 <a foo=bar !>
538 """
539 for pat, a in A_NAME_LEX_COMPILED:
540 m = pat.match(self.s, self.pos)
541 if m:
542 self.pos = m.end(0) # Advance
543
544 if a == attr_name.Ok:
545 #log('%r', m.groups())
546 self.name_start = m.start(1)
547 self.name_end = m.end(1)
548 # Set state based on =
549 if m.group(2) is None:
550 self.next_value_is_missing = True
551 return attr_name.Ok, self.name_start, self.name_end
552 else:
553 # Reset state - e.g. you must call AttrNameEquals
554 self.name_start = -1
555 self.name_end = -1
556 self.next_value_is_missing = False
557
558 if a == attr_name.Invalid:
559 return attr_name.Invalid, -1, -1
560 if a == attr_name.Done:
561 return attr_name.Done, -1, -1
562 else:
563 raise AssertionError('h8_id.Invalid rule should have matched')
564
565 def _CanonicalAttrName(self):
566 # type: () -> str
567 assert self.name_start >= 0, self.name_start
568 assert self.name_end >= 0, self.name_end
569
570 attr_name = self.s[self.name_start:self.name_end]
571 if attr_name.islower():
572 return attr_name
573 else:
574 return attr_name.lower()
575
576 def AttrNameEquals(self, expected):
577 # type: (str) -> bool
578 """
579 TODO: Must call this after ReadName() ?
580 Because that can FAIL.
581 """
582 return expected == self._CanonicalAttrName()
583
584 def _QuotedRead(self):
585 # type: () -> Tuple[h8_id, end_pos]
586
587 for pat, tok_id in QUOTED_VALUE_LEX_COMPILED:
588 m = pat.match(self.s, self.pos)
589 if m:
590 end_pos = m.end(0) # Advance
591 log('_QuotedRead %r', self.s[self.pos:end_pos])
592 return tok_id, end_pos
593 else:
594 context = self.s[self.pos:self.pos + 10]
595 raise AssertionError('h8_id.Invalid rule should have matched %r' %
596 context)
597
598 def ReadRawValue(self):
599 # type: () -> Tuple[attr_value_t, int, int]
600 """Read the attribute value.
601
602 In general, it is escaped or "raw"
603
604 Can only be called after a SUCCESSFUL ReadName().
605 Assuming ReadName() returned a value, this should NOT fail.
606 """
607 # ReadName() invariant
608 assert self.name_start >= 0, self.name_start
609 assert self.name_end >= 0, self.name_end
610
611 self.name_start = -1
612 self.name_end = -1
613
614 if self.next_value_is_missing:
615 return attr_value_e.Missing, -1, -1
616
617 # Now read " ', unquoted or empty= is valid too.
618 for pat, a in A_VALUE_LEX_COMPILED:
619 m = pat.match(self.s, self.pos)
620 if m:
621 self.pos = m.end(0) # Advance
622
623 #log('m %s', m.groups())
624 if a == h8_val_id.UnquotedVal:
625 return attr_value_e.Unquoted, m.start(0), m.end(0)
626
627 if a == h8_val_id.DoubleQuote:
628 left_inner = self.pos
629 while True:
630 tok_id, q_end_pos = self._QuotedRead()
631 if tok_id == h8_id.Invalid:
632 raise LexError(self.s, self.pos)
633 if tok_id == h8_id.DoubleQuote:
634 return attr_value_e.DoubleQuoted, left_inner, self.pos
635 self.pos = q_end_pos # advance
636
637 if a == h8_val_id.SingleQuote:
638
639 left_inner = self.pos
640 while True:
641 tok_id, q_end_pos = self._QuotedRead()
642 if tok_id == h8_id.Invalid:
643 raise LexError(self.s, self.pos)
644 if tok_id == h8_id.SingleQuote:
645 return attr_value_e.SingleQuoted, left_inner, self.pos
646 self.pos = q_end_pos # advance
647
648 if a == h8_val_id.NoMatch:
649 # <a foo = >
650 return attr_value_e.Empty, -1, -1
651 else:
652 raise AssertionError('h8_val_id.NoMatch rule should have matched')
653
654 def SkipValue(self):
655 # type: () -> None
656 # Just ignore it and return
657 self.ReadRawValue()
658
659 def ReadValueAndDecode(self):
660 # type: () -> str
661 """Read the attribute vlaue
662 """
663 # TODO: tokenize it
664 pass
665
666
667# Tag names:
668# Match <a or </a
669# Match <h2, but not <2h
670#
671# HTML 5 doesn't restrict tag names at all
672# https://html.spec.whatwg.org/#toc-syntax
673#
674# XML allows : - .
675# https://www.w3.org/TR/xml/#NT-NameChar
676
677# Namespaces for MathML, SVG
678# XLink, XML, XMLNS
679#
680# https://infra.spec.whatwg.org/#namespaces
681#
682# Allow - for td-attrs
683
684# TODO: we don't need to capture the tag name here? That's done at the top
685# level
686_TAG_RE = re.compile(r'/? \s* (%s)' % _NAME, re.VERBOSE)
687
688_TAG_LAST_RE = re.compile(r'\s* /? >', re.VERBOSE)
689
690# To match href="foo"
691# Note: in HTML5 and XML, single quoted attributes are also valid
692
693# <button disabled> is standard usage
694
695# NOTE: This used to allow whitespace around =
696# <a foo = "bar"> makes sense in XML
697# But then you also have
698# <a foo= bar> - which is TWO attributes, in HTML5
699# So the space is problematic
700
701_ATTR_RE = re.compile(
702 r'''
703\s+ # Leading whitespace is required
704(%s) # Attribute name
705(?: # Optional attribute value
706 \s* = \s* # Spaces allowed around =
707 (?:
708 " ([^>"\x00]*) " # double quoted value
709 | ' ([^>'\x00]*) ' # single quoted value
710 | (%s) # Attribute value
711 )
712)?
713''' % (_NAME, _UNQUOTED_VALUE_OLD), re.VERBOSE)
714
715
716class TagLexer(object):
717 """
718 Given a tag like <a href="..."> or <link type="..." />, the TagLexer
719 provides a few operations:
720
721 - What is the tag?
722 - Iterate through the attributes, giving (name, value_start_pos, value_end_pos)
723 """
724
725 def __init__(self, s):
726 # type: (str) -> None
727 self.s = s
728 self.start_pos = -1 # Invalid
729 self.end_pos = -1
730
731 def Reset(self, start_pos, end_pos):
732 # type: (int, int) -> None
733 """Reuse instances of this object."""
734 assert start_pos >= 0, start_pos
735 assert end_pos >= 0, end_pos
736
737 self.start_pos = start_pos
738 self.end_pos = end_pos
739
740 def WholeTagString(self):
741 # type: () -> str
742 """Return the entire tag string, e.g. <a href='foo'>"""
743 return self.s[self.start_pos:self.end_pos]
744
745 def GetTagName(self):
746 # type: () -> str
747 # First event
748 tok_id, start, end = next(self.Tokens())
749 return self.s[start:end]
750
751 def GetSpanForAttrValue(self, attr_name):
752 # type: (str) -> Tuple[int, int]
753 """
754 Used by oils_doc.py, for href shortcuts
755 """
756 # Algorithm: search for QuotedValue or UnquotedValue after AttrName
757 # TODO: Could also cache these
758
759 events = self.Tokens()
760 val = (-1, -1)
761 try:
762 while True:
763 tok_id, start, end = next(events)
764 if tok_id == h8_tag_id.AttrName:
765 name = self.s[start:end]
766 if name == attr_name:
767 # The value should come next
768 tok_id, start, end = next(events)
769 assert tok_id in (
770 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
771 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
772 val = start, end
773 break
774
775 except StopIteration:
776 pass
777 return val
778
779 def GetAttrRaw(self, attr_name):
780 # type: (str) -> Optional[str]
781 """
782 Return the value, which may be UNESCAPED.
783 """
784 start, end = self.GetSpanForAttrValue(attr_name)
785 if start == -1:
786 return None
787 return self.s[start:end]
788
789 def AllAttrsRawSlice(self):
790 # type: () -> List[Tuple[str, int, int]]
791 """
792 Get a list of pairs [('class', 3, 5), ('href', 9, 12)]
793 """
794 slices = []
795 events = self.Tokens()
796 try:
797 while True:
798 tok_id, start, end = next(events)
799 if tok_id == h8_tag_id.AttrName:
800 name = self.s[start:end]
801
802 # The value should come next
803 tok_id, start, end = next(events)
804 assert tok_id in (
805 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
806 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
807 # Note: quoted values may have &amp;
808 # We would need ANOTHER lexer to unescape them, but we
809 # don't need that for ul-table
810 slices.append((name, start, end))
811 except StopIteration:
812 pass
813 return slices
814
815 def AllAttrsRaw(self):
816 # type: () -> List[Tuple[str, str]]
817 """
818 Get a list of pairs [('class', 'foo'), ('href', '?foo=1&amp;bar=2')]
819
820 The quoted values may be escaped. We would need another lexer to
821 unescape them.
822 """
823 slices = self.AllAttrsRawSlice()
824 pairs = []
825 for name, start, end in slices:
826 pairs.append((name, self.s[start:end]))
827 return pairs
828
829 def Tokens(self):
830 # type: () -> Iterator[Tuple[h8_tag_id_t, int, int]]
831 """
832 Yields a sequence of tokens: Tag (AttrName AttrValue?)*
833
834 Where each Token is (Type, start_pos, end_pos)
835
836 Note that start and end are NOT redundant! We skip over some unwanted
837 characters.
838 """
839 m = _TAG_RE.match(self.s, self.start_pos + 1)
840 if not m:
841 raise RuntimeError("Couldn't find HTML tag in %r" %
842 self.WholeTagString())
843 yield h8_tag_id.TagName, m.start(1), m.end(1)
844
845 pos = m.end(0)
846 #log('POS %d', pos)
847
848 while True:
849 # don't search past the end
850 m = _ATTR_RE.match(self.s, pos, self.end_pos)
851 if not m:
852 #log('BREAK pos %d', pos)
853 break
854 #log('AttrName %r', m.group(1))
855
856 yield h8_tag_id.AttrName, m.start(1), m.end(1)
857
858 #log('m.groups() %r', m.groups())
859 if m.group(2) is not None:
860 # double quoted
861 yield h8_tag_id.QuotedValue, m.start(2), m.end(2)
862 elif m.group(3) is not None:
863 # single quoted - TODO: could have different token types
864 yield h8_tag_id.QuotedValue, m.start(3), m.end(3)
865 elif m.group(4) is not None:
866 yield h8_tag_id.UnquotedValue, m.start(4), m.end(4)
867 else:
868 # <button disabled>
869 end = m.end(0)
870 yield h8_tag_id.MissingValue, end, end
871
872 # Skip past the "
873 pos = m.end(0)
874
875 #log('TOK %r', self.s)
876
877 m = _TAG_LAST_RE.match(self.s, pos)
878 #log('_TAG_LAST_RE match %r', self.s[pos:])
879 if not m:
880 # Extra data at end of tag. TODO: add messages for all these.
881 raise LexError(self.s, pos)
882
883
884# This is similar but not identical to
885# " ([^>"\x00]*) " # double quoted value
886# | ' ([^>'\x00]*) ' # single quoted value
887#
888# Note: for unquoted values, & isn't allowed, and thus &amp; and &#99; and
889# &#x99; are not allowed. We could relax that?
890ATTR_VALUE_LEX = CHAR_LEX + [
891 (r'[^>&\x00]+', h8_id.RawData),
892 (r'.', h8_id.Invalid),
893]
894
895ATTR_VALUE_LEX_COMPILED = MakeLexer(ATTR_VALUE_LEX)
896
897QUOTED_VALUE_LEX = CHAR_LEX + [
898 (r'"', h8_id.DoubleQuote),
899 (r"'", h8_id.SingleQuote),
900 (r'<', h8_id.BadLessThan), # BadAmpersand is in CharLex
901 (r'''[^"'<>&\x00]+''', h8_id.RawData),
902 # This includes > - it is not BadGreaterThan because it's NOT recoverable
903 (r'.', h8_id.Invalid),
904]
905
906QUOTED_VALUE_LEX_COMPILED = MakeLexer(QUOTED_VALUE_LEX)
907
908
909class AttrValueLexer(object):
910 """
911 <a href="foo=99&amp;bar">
912 <a href='foo=99&amp;bar'>
913 <a href=unquoted>
914 """
915
916 def __init__(self, s):
917 # type: (str) -> None
918 self.s = s
919 self.start_pos = -1 # Invalid
920 self.end_pos = -1
921
922 def Reset(self, start_pos, end_pos):
923 # type: (int, int) -> None
924 """Reuse instances of this object."""
925 assert start_pos >= 0, start_pos
926 assert end_pos >= 0, end_pos
927
928 self.start_pos = start_pos
929 self.end_pos = end_pos
930
931 def NumTokens(self):
932 # type: () -> int
933 num_tokens = 0
934 pos = self.start_pos
935 for tok_id, end_pos in self.Tokens():
936 if tok_id == h8_id.Invalid:
937 raise LexError(self.s, pos)
938 pos = end_pos
939 #log('pos %d', pos)
940 num_tokens += 1
941 return num_tokens
942
943 def Tokens(self):
944 # type: () -> Iterator[Tuple[h8_id_t, int]]
945 pos = self.start_pos
946 while pos < self.end_pos:
947 # Find the first match, like above.
948 # Note: frontend/match.py uses _LongestMatch(), which is different!
949 # TODO: reconcile them. This lexer should be expressible in re2c.
950 for pat, tok_id in ATTR_VALUE_LEX_COMPILED:
951 m = pat.match(self.s, pos)
952 if m:
953 if 0:
954 tok_str = m.group(0)
955 log('token = %r', tok_str)
956
957 end_pos = m.end(0)
958 yield tok_id, end_pos
959 pos = end_pos
960 break
961 else:
962 raise AssertionError('h8_id.Invalid rule should have matched')