OILS / data_lang / htm8.py View on Github | oils.pub

960 lines, 399 significant
1"""data_lang/htm8.py
2
3TODO
4
5API:
6- Get rid of AttrValueLexer - this should be in the TagLexer
7 - this also means that unquoted values can be more similar
8 - We can use a single lexer mode for everything inside <>
9 - the SPACE is the only difference
10- Deprecate tag_lexer.GetTagName() in favor of lx.CanonicalTagName() or
11 _LiteralTagName()
12- UTF-8 check, like JSON8
13- re2c
14 - port lexer, which will fix static typing issues
15 - the abstraction needs to support submatch?
16 - for finding the end of a tag, etc.?
17
18- LexError and ParseError need details
19 - harmonize with data_lang/j8.py, which uses error.Decode(msg, ...,
20 cur_line_num)
21
22- Copy all errors into doc/ref/chap-errors.md
23 - This helps understand the language
24
25- Update doc/htm8.md
26- list of Algorithms:
27 - lex just the top level
28 - lex both levels
29 - and match tags - this is the level for value.Htm8Frag?
30 - convert to XML!
31 - lazy selection by tag, or attr (id= and class=)
32 - lazy selection by CSS selector expression
33 - convert to DOMTree
34 - sed-like replacement of DOM Tree or element
35 - untrusted HTML filter, e.g. like StackOverflow / Reddit
36 - this is Safe HTM8
37 - should have a zero alloc way to support this, with good errors?
38 - I think most of them silently strip data
39"""
40
41import re
42
43from typing import Dict, List, Tuple, Optional, IO, Iterator, Any
44
45from _devbuild.gen.htm8_asdl import (h8_id, h8_id_t, h8_tag_id, h8_tag_id_t,
46 h8_tag_id_str, attr_name, attr_name_t,
47 attr_value_e, attr_value_t, h8_val_id)
48from doctools.util import log
49
50
51class LexError(Exception):
52 """
53 Examples of lex errors:
54
55 - h8_id.Invalid, like <> or &&
56 - Unclosed <!-- <? <![CDATA[ <script> <style>
57 """
58
59 def __init__(self, s, start_pos):
60 # type: (str, int) -> None
61 self.s = s
62 self.start_pos = start_pos
63
64 def __str__(self):
65 # type: () -> str
66 return '(LexError %r)' % (self.s[self.start_pos:self.start_pos + 20])
67
68
69def _FindLineNum(s, error_pos):
70 # type: (str, int) -> int
71 current_pos = 0
72 line_num = 1
73 while True:
74 newline_pos = s.find('\n', current_pos)
75 #log('current = %d, N %d, line %d', current_pos, newline_pos, line_num)
76
77 if newline_pos == -1: # this is the last line
78 return line_num
79 if newline_pos >= error_pos:
80 return line_num
81 line_num += 1
82 current_pos = newline_pos + 1
83
84
85class ParseError(Exception):
86 """
87 Examples of parse errors
88
89 - unbalanced tag structure
90 - ul_table.py errors
91 """
92
93 def __init__(self, msg, s=None, start_pos=-1):
94 # type: (str, Optional[str], int) -> None
95 self.msg = msg
96 self.s = s
97 self.start_pos = start_pos
98
99 def __str__(self):
100 # type: () -> str
101 if self.s is not None:
102 assert self.start_pos != -1, self.start_pos
103 snippet = (self.s[self.start_pos:self.start_pos + 20])
104
105 line_num = _FindLineNum(self.s, self.start_pos)
106 else:
107 snippet = ''
108 line_num = -1
109 msg = 'line %d: %r %r' % (line_num, self.msg, snippet)
110 return msg
111
112
113class Output(object):
114 """Takes an underlying input buffer and an output file. Maintains a
115 position in the input buffer.
116
117 Print FROM the input or print new text to the output.
118 """
119
120 def __init__(self, s, f, left_pos=0, right_pos=-1):
121 # type: (str, IO[str], int, int) -> None
122 self.s = s
123 self.f = f
124 self.pos = left_pos
125 self.right_pos = len(s) if right_pos == -1 else right_pos
126
127 def SkipTo(self, pos):
128 # type: (int) -> None
129 """Skip to a position."""
130 self.pos = pos
131
132 def PrintUntil(self, pos):
133 # type: (int) -> None
134 """Print until a position."""
135 piece = self.s[self.pos:pos]
136 self.f.write(piece)
137 self.pos = pos
138
139 def PrintTheRest(self):
140 # type: () -> None
141 """Print until the end of the string."""
142 self.PrintUntil(self.right_pos)
143
144 def Print(self, s):
145 # type: (str) -> None
146 """Print text to the underlying buffer."""
147 self.f.write(s)
148
149
150def MakeLexer(rules):
151 return [(re.compile(pat, re.VERBOSE), i) for (pat, i) in rules]
152
153
154#
155# Eggex
156#
157# Tag = / ~['>']+ /
158
159# Is this valid? A single character?
160# Tag = / ~'>'* /
161
162# Maybe better: / [NOT '>']+/
163# capital letters not allowed there?
164#
165# But then this is confusing:
166# / [NOT ~digit]+/
167#
168# / [NOT digit] / is [^\d]
169# / ~digit / is \D
170#
171# Or maybe:
172#
173# / [~ digit]+ /
174# / [~ '>']+ /
175# / [NOT '>']+ /
176
177# End = / '</' Tag '>' /
178# StartEnd = / '<' Tag '/>' /
179# Start = / '<' Tag '>' /
180#
181# EntityRef = / '&' dot{* N} ';' /
182
183# Tag name, or attribute name
184# colon is used in XML
185
186# https://www.w3.org/TR/xml/#NT-Name
187# Hm there is a lot of unicode stuff. We are simplifying parsing
188
189_NAME = r'[a-zA-Z][a-zA-Z0-9:_\-]*' # must start with letter
190
191CHAR_LEX = [
192 # Characters
193 # https://www.w3.org/TR/xml/#sec-references
194 (r'&\# [0-9]+ ;', h8_id.DecChar),
195 (r'&\# x[0-9a-fA-F]+ ;', h8_id.HexChar),
196 (r'& %s ;' % _NAME, h8_id.CharEntity),
197 # Allow unquoted, and quoted
198 (r'&', h8_id.BadAmpersand),
199]
200
201HTM8_LEX = CHAR_LEX + [
202 # TODO: CommentBegin, ProcessingBegin, CDataBegin could have an additional
203 # action associated with them? The ending substring
204 (r'<!--', h8_id.CommentBegin),
205
206 # Processing instruction are used for the XML header:
207 # <?xml version="1.0" encoding="UTF-8"?>
208 # They are technically XML-only, but in HTML5, they are another kind of
209 # comment:
210 #
211 # https://developer.mozilla.org/en-US/docs/Web/API/ProcessingInstruction
212 #
213 (r'<\?', h8_id.ProcessingBegin),
214 # Not necessary in HTML5, but occurs in XML
215 (r'<!\[CDATA\[', h8_id.CDataBegin), # <![CDATA[
216
217 # Markup declarations
218 # - In HTML5, there is only <!DOCTYPE html>
219 # - XML has 4 more declarations: <!ELEMENT ...> ATTLIST ENTITY NOTATION
220 # - these seem to be part of DTD
221 # - it's useful to skip these, and be able to parse the rest of the document
222 # - Note: < is allowed?
223 (r'<! [^>\x00]+ >', h8_id.Decl),
224
225 # Tags
226 # Notes:
227 # - We look for a valid tag name, but we don't validate attributes.
228 # That's done in the tag lexer.
229 # - We don't allow leading whitespace
230 (r'</ (%s) >' % _NAME, h8_id.EndTag),
231 # self-closing <br/> comes before StartTag
232 # could/should these be collapsed into one rule?
233 (r'< (%s) [^>\x00]* />' % _NAME, h8_id.StartEndTag), # end </a>
234 (r'< (%s) [^>\x00]* >' % _NAME, h8_id.StartTag), # start <a>
235
236 # HTML5 allows unescaped > in raw data, but < is not allowed.
237 # https://stackoverflow.com/questions/10462348/right-angle-bracket-in-html
238 #
239 # - My early blog has THREE errors when disallowing >
240 # - So do some .wwz files
241 (r'[^&<>\x00]+', h8_id.RawData),
242 (r'>', h8_id.BadGreaterThan),
243 # NUL is the end, an accomodation for re2c. Like we do in frontend/match.
244 (r'\x00', h8_id.EndOfStream),
245 # This includes < - it is not BadLessThan because it's NOT recoverable
246 (r'.', h8_id.Invalid),
247]
248
249# Old notes:
250#
251# Non-greedy matches are regular and can be matched in linear time
252# with RE2.
253#
254# https://news.ycombinator.com/item?id=27099798
255#
256
257# This person tried to do it with a regex:
258#
259# https://skeptric.com/html-comment-regexp/index.html
260
261# . is any char except newline
262# https://re2c.org/manual/manual_c.html
263
264# Discarded options
265#(r'<!-- .*? -->', h8_id.Comment),
266
267# Hack from Claude: \s\S instead of re.DOTALL. I don't like this
268#(r'<!-- [\s\S]*? -->', h8_id.Comment),
269#(r'<!-- (?:.|[\n])*? -->', h8_id.Comment),
270
271HTM8_LEX_COMPILED = MakeLexer(HTM8_LEX)
272
273
274class Lexer(object):
275
276 def __init__(self, s, left_pos=0, right_pos=-1, no_special_tags=False):
277 # type: (str, int, int, bool) -> None
278 self.s = s
279 self.pos = left_pos
280 self.right_pos = len(s) if right_pos == -1 else right_pos
281 self.no_special_tags = no_special_tags
282
283 # string -> compiled regex pattern object
284 self.cache = {} # type: Dict[str, Any]
285
286 # either </script> or </style> - we search until we see that
287 self.search_state = None # type: Optional[str]
288
289 # Position of tag name, if applicable
290 # - Set after you get a StartTag, EndTag, or StartEndTag
291 # - Unset on other tags
292 self.tag_pos_left = -1
293 self.tag_pos_right = -1
294
295 def _Read(self):
296 # type: () -> Tuple[h8_id_t, int]
297 if self.pos == self.right_pos:
298 return h8_id.EndOfStream, self.pos
299
300 assert self.pos < self.right_pos, self.pos
301
302 if self.search_state is not None and not self.no_special_tags:
303 # TODO: case-insensitive search for </SCRIPT> <SCRipt> ?
304 #
305 # Another strategy: enter a mode where we find ONLY the end tag
306 # regex, and any data that's not <, and then check the canonical
307 # tag name for 'script' or 'style'.
308 pos = self.s.find(self.search_state, self.pos)
309 if pos == -1:
310 # unterminated <script> or <style>
311 raise LexError(self.s, self.pos)
312 self.search_state = None
313 # beginning
314 return h8_id.HtmlCData, pos
315
316 # Find the first match.
317 # Note: frontend/match.py uses _LongestMatch(), which is different!
318 # TODO: reconcile them. This lexer should be expressible in re2c.
319
320 for pat, tok_id in HTM8_LEX_COMPILED:
321 m = pat.match(self.s, self.pos)
322 if m:
323 if tok_id in (h8_id.StartTag, h8_id.EndTag, h8_id.StartEndTag):
324 self.tag_pos_left = m.start(1)
325 self.tag_pos_right = m.end(1)
326 else:
327 # Reset state
328 self.tag_pos_left = -1
329 self.tag_pos_right = -1
330
331 if tok_id == h8_id.CommentBegin:
332 pos = self.s.find('-->', self.pos)
333 if pos == -1:
334 # unterminated <!--
335 raise LexError(self.s, self.pos)
336 return h8_id.Comment, pos + 3 # -->
337
338 if tok_id == h8_id.ProcessingBegin:
339 pos = self.s.find('?>', self.pos)
340 if pos == -1:
341 # unterminated <?
342 raise LexError(self.s, self.pos)
343 return h8_id.Processing, pos + 2 # ?>
344
345 if tok_id == h8_id.CDataBegin:
346 pos = self.s.find(']]>', self.pos)
347 if pos == -1:
348 # unterminated <![CDATA[
349 raise LexError(self.s, self.pos)
350 return h8_id.CData, pos + 3 # ]]>
351
352 if tok_id == h8_id.StartTag:
353 # TODO: reduce allocations
354 if (self.TagNameEquals('script') or
355 self.TagNameEquals('style')):
356 # <SCRipt a=b> -> </SCRipt>
357 self.search_state = '</' + self._LiteralTagName() + '>'
358
359 return tok_id, m.end()
360 else:
361 raise AssertionError('h8_id.Invalid rule should have matched')
362
363 def TagNamePos(self):
364 """The right position of the tag pos"""
365 assert self.tag_pos_right != -1, self.tag_pos_right
366 return self.tag_pos_right
367
368 def TagNameEquals(self, expected):
369 # type: (str) -> bool
370 assert self.tag_pos_left != -1, self.tag_pos_left
371 assert self.tag_pos_right != -1, self.tag_pos_right
372
373 # TODO: In C++, this does not need an allocation. Can we test
374 # directly?
375 return expected == self.CanonicalTagName()
376
377 def _LiteralTagName(self):
378 # type: () -> str
379 assert self.tag_pos_left != -1, self.tag_pos_left
380 assert self.tag_pos_right != -1, self.tag_pos_right
381
382 return self.s[self.tag_pos_left:self.tag_pos_right]
383
384 def CanonicalTagName(self):
385 # type: () -> str
386 tag_name = self._LiteralTagName()
387 # Most tags are already lower case, so avoid allocation with this conditional
388 # TODO: this could go in the mycpp runtime?
389 if tag_name.islower():
390 return tag_name
391 else:
392 return tag_name.lower()
393
394 def Read(self):
395 # type: () -> Tuple[h8_id_t, int]
396 tok_id, end_pos = self._Read()
397 self.pos = end_pos # advance
398 return tok_id, end_pos
399
400 def LookAhead(self, regex):
401 # type: (str) -> bool
402 """
403 Currently used for ul_table.py. But taking a dynamic regex string is
404 not the right interface.
405 """
406 # Cache the regex compilation. This could also be LookAheadFor(THEAD)
407 # or something.
408 pat = self.cache.get(regex)
409 if pat is None:
410 pat = re.compile(regex)
411 self.cache[regex] = pat
412
413 m = pat.match(self.s, self.pos)
414 return m is not None
415
416
417A_NAME_LEX = [
418 # Leading whitespace is required, to separate attributes.
419 #
420 # If the = is not present, then we set the lexer in a state for
421 # attr_value_e.Missing.
422 (r'\s+ (%s) \s* (=)? \s*' % _NAME, attr_name.Ok),
423 # unexpected EOF
424
425 # The closing > or /> is treated as end of stream, and it's not an error.
426 (r'\s* /? >', attr_name.Done),
427
428 # NUL should not be possible, because the top-level
429
430 # This includes < - it is not BadLessThan because it's NOT recoverable
431 (r'.', attr_name.Invalid),
432]
433
434A_NAME_LEX_COMPILED = MakeLexer(A_NAME_LEX)
435
436# Here we just loop on regular tokens
437#
438# Examples:
439# <a href = unquoted&amp;foo >
440# <a href = unquoted&foo > # BadAmpersand is allowed I guess
441# <a href ="unquoted&foo" > # double quoted
442# <a href ='unquoted&foo' > # single quoted
443# <a href = what"foo" > # HTML5 allows this, but we could disallow it if
444# it's not common. It opens up the j"" and $"" extensions
445# <a href = what'foo' > # ditto
446#
447# Problem: <a href=foo/> - this is hard to recognize
448# Because is the unquoted value "foo/" or "foo" ?
449
450# Be very lenient - just no whitespace or special HTML chars
451# I don't think this is more lenient than HTML5, though we should check.
452#
453# Bug fix: Also disallow /
454
455# TODO: get rid of OLD copy
456_UNQUOTED_VALUE_OLD = r'''[^ \t\r\n<>&/"'\x00]*'''
457_UNQUOTED_VALUE = r'''[^ \t\r\n<>&/"'\x00]+'''
458
459# Restrictive definition, similar to _NAME
460# I was trying to capture #ble.sh and so forth
461# I also had unquoted //github.com, etc.
462
463# _UNQUOTED_VALUE = r'''[a-zA-Z0-9:_\-]+'''
464#
465# For now, I guess we live with <a href=?foo/>
466
467# What comes after = ?
468A_VALUE_LEX = [
469 (r'"', h8_val_id.DoubleQuote),
470 (r"'", h8_val_id.SingleQuote),
471 (_UNQUOTED_VALUE, h8_val_id.UnquotedVal),
472
473 #(r'[ \r\n\t]', h8_id.Whitespace), # terminates unquoted values
474 #(r'[^ \r\n\t&>\x00]', h8_id.RawData),
475 #(r'[>\x00]', h8_id.EndOfStream),
476 # e.g. < is an error
477 (r'.', h8_val_id.NoMatch),
478]
479
480A_VALUE_LEX_COMPILED = MakeLexer(A_VALUE_LEX)
481
482# What's inside "" or '' ?
483QUOTED_VALUE_LEX = CHAR_LEX + [
484 (r'"', h8_id.DoubleQuote),
485 (r"'", h8_id.SingleQuote),
486 (r'<', h8_id.BadLessThan), # BadAmpersand is in CharLex
487 (r'''[^"'<>&\x00]+''', h8_id.RawData),
488 # This includes > - it is not BadGreaterThan because it's NOT recoverable
489 (r'.', h8_id.Invalid),
490]
491
492QUOTED_VALUE_LEX_COMPILED = MakeLexer(QUOTED_VALUE_LEX)
493
494
495class AttrLexer(object):
496 """
497 Typical usage:
498
499 while True:
500 n, start_pos, end_pos = attr_lx.ReadName()
501 if n == attr_name.Ok:
502 if attr_lx.AttrNameEquals('div'):
503 print('div')
504
505 # TODO: also pass Optional[List[]] out_tokens?
506 v, start_pos, end_pos = attr_lx.ReadValue()
507 """
508
509 def __init__(self, s):
510 # type: (str) -> None
511 self.s = s
512 self.tag_name_pos = -1 # Invalid
513 self.tag_end_pos = -1
514 self.pos = -1
515
516 self.name_start = -1
517 self.name_end = -1
518 self.next_value_is_missing = False
519
520 def Init(self, tag_name_pos, end_pos):
521 # type: (int, int) -> None
522 """Initialize so we can read names and values.
523
524 Example:
525 'x <a y>' # tag_name_pos=4, end_pos=6
526 'x <a>' # tag_name_pos=4, end_pos=4
527
528 The Reset() method is used to reuse instances of the AttrLexer object.
529 """
530 assert tag_name_pos >= 0, tag_name_pos
531 assert end_pos >= 0, end_pos
532
533 log('TAG NAME POS %d', tag_name_pos)
534
535 self.tag_name_pos = tag_name_pos
536 self.end_pos = end_pos
537
538 self.pos = tag_name_pos
539
540 def ReadName(self):
541 # type: () -> Tuple[attr_name_t, int, int]
542 """Reads the attribute name
543
544 EOF case:
545 <a>
546 <a >
547
548 Error case:
549 <a !>
550 <a foo=bar !>
551 """
552 for pat, a in A_NAME_LEX_COMPILED:
553 m = pat.match(self.s, self.pos)
554 if m:
555 self.pos = m.end(0) # Advance
556
557 if a == attr_name.Ok:
558 #log('%r', m.groups())
559 self.name_start = m.start(1)
560 self.name_end = m.end(1)
561 # Set state based on =
562 if m.group(2) is None:
563 self.next_value_is_missing = True
564 return attr_name.Ok, self.name_start, self.name_end
565 else:
566 # Reset state - e.g. you must call AttrNameEquals
567 self.name_start = -1
568 self.name_end = -1
569 self.next_value_is_missing = False
570
571 if a == attr_name.Invalid:
572 return attr_name.Invalid, -1, -1
573 if a == attr_name.Done:
574 return attr_name.Done, -1, -1
575 else:
576 raise AssertionError('h8_id.Invalid rule should have matched')
577
578 def _CanonicalAttrName(self):
579 # type: () -> str
580 assert self.name_start >= 0, self.name_start
581 assert self.name_end >= 0, self.name_end
582
583 attr_name = self.s[self.name_start:self.name_end]
584 if attr_name.islower():
585 return attr_name
586 else:
587 return attr_name.lower()
588
589 def AttrNameEquals(self, expected):
590 # type: (str) -> bool
591 """
592 TODO: Must call this after ReadName() ?
593 Because that can FAIL.
594 """
595 return expected == self._CanonicalAttrName()
596
597 def _QuotedRead(self):
598 # type: () -> Tuple[h8_id_t, int]
599
600 for pat, tok_id in QUOTED_VALUE_LEX_COMPILED:
601 m = pat.match(self.s, self.pos)
602 if m:
603 end_pos = m.end(0) # Advance
604 #log('_QuotedRead %r', self.s[self.pos:end_pos])
605 return tok_id, end_pos
606 else:
607 context = self.s[self.pos:self.pos + 10]
608 raise AssertionError('h8_id.Invalid rule should have matched %r' %
609 context)
610
611 def ReadValue(self, tokens_out=None):
612 # type: (Optional[List[Tuple[h8_id, int]]]) -> Tuple[attr_value_t, int, int]
613 """Read the attribute value.
614
615 In general, it is escaped or "raw"
616
617 Can only be called after a SUCCESSFUL ReadName().
618 Assuming ReadName() returned a value, this should NOT fail.
619 """
620 # ReadName() invariant
621 assert self.name_start >= 0, self.name_start
622 assert self.name_end >= 0, self.name_end
623
624 self.name_start = -1
625 self.name_end = -1
626
627 if self.next_value_is_missing:
628 return attr_value_e.Missing, -1, -1
629
630 # Now read " ', unquoted or empty= is valid too.
631 for pat, a in A_VALUE_LEX_COMPILED:
632 m = pat.match(self.s, self.pos)
633 if m:
634 self.pos = m.end(0) # Advance
635
636 #log('m %s', m.groups())
637
638 # Note: Unquoted value can't contain &amp; etc. now, so there
639 # is no unquoting, and no respecting tokens_raw.
640 if a == h8_val_id.UnquotedVal:
641 return attr_value_e.Unquoted, m.start(0), m.end(0)
642
643 # TODO: respect tokens_out
644 if a == h8_val_id.DoubleQuote:
645 left_inner = self.pos
646 while True:
647 tok_id, q_end_pos = self._QuotedRead()
648 if tok_id == h8_id.Invalid:
649 raise LexError(self.s, self.pos)
650 if tok_id == h8_id.DoubleQuote:
651 return attr_value_e.DoubleQuoted, left_inner, self.pos
652 self.pos = q_end_pos # advance
653
654 # TODO: respect tokens_out
655 if a == h8_val_id.SingleQuote:
656 left_inner = self.pos
657 while True:
658 tok_id, q_end_pos = self._QuotedRead()
659 if tok_id == h8_id.Invalid:
660 raise LexError(self.s, self.pos)
661 if tok_id == h8_id.SingleQuote:
662 return attr_value_e.SingleQuoted, left_inner, self.pos
663 self.pos = q_end_pos # advance
664
665 if a == h8_val_id.NoMatch:
666 # <a foo = >
667 return attr_value_e.Empty, -1, -1
668 else:
669 raise AssertionError('h8_val_id.NoMatch rule should have matched')
670
671
672#
673# OLD API - REMOVE THIS
674#
675
676# Tag names:
677# Match <a or </a
678# Match <h2, but not <2h
679#
680# HTML 5 doesn't restrict tag names at all
681# https://html.spec.whatwg.org/#toc-syntax
682#
683# XML allows : - .
684# https://www.w3.org/TR/xml/#NT-NameChar
685
686# Namespaces for MathML, SVG
687# XLink, XML, XMLNS
688#
689# https://infra.spec.whatwg.org/#namespaces
690#
691# Allow - for td-attrs
692
693# TODO: we don't need to capture the tag name here? That's done at the top
694# level
695_TAG_RE = re.compile(r'/? \s* (%s)' % _NAME, re.VERBOSE)
696
697_TAG_LAST_RE = re.compile(r'\s* /? >', re.VERBOSE)
698
699# To match href="foo"
700# Note: in HTML5 and XML, single quoted attributes are also valid
701
702# <button disabled> is standard usage
703
704# NOTE: This used to allow whitespace around =
705# <a foo = "bar"> makes sense in XML
706# But then you also have
707# <a foo= bar> - which is TWO attributes, in HTML5
708# So the space is problematic
709
710_ATTR_RE = re.compile(
711 r'''
712\s+ # Leading whitespace is required
713(%s) # Attribute name
714(?: # Optional attribute value
715 \s* = \s* # Spaces allowed around =
716 (?:
717 " ([^>"\x00]*) " # double quoted value
718 | ' ([^>'\x00]*) ' # single quoted value
719 | (%s) # Attribute value
720 )
721)?
722''' % (_NAME, _UNQUOTED_VALUE_OLD), re.VERBOSE)
723
724
725class TagLexer(object):
726 """
727 Given a tag like <a href="..."> or <link type="..." />, the TagLexer
728 provides a few operations:
729
730 - What is the tag?
731 - Iterate through the attributes, giving (name, value_start_pos, value_end_pos)
732 """
733
734 def __init__(self, s):
735 # type: (str) -> None
736 self.s = s
737 self.start_pos = -1 # Invalid
738 self.end_pos = -1
739
740 def Reset(self, start_pos, end_pos):
741 # type: (int, int) -> None
742 """Reuse instances of this object."""
743 assert start_pos >= 0, start_pos
744 assert end_pos >= 0, end_pos
745
746 self.start_pos = start_pos
747 self.end_pos = end_pos
748
749 def WholeTagString(self):
750 # type: () -> str
751 """Return the entire tag string, e.g. <a href='foo'>"""
752 return self.s[self.start_pos:self.end_pos]
753
754 def GetTagName(self):
755 # type: () -> str
756 # First event
757 tok_id, start, end = next(self.Tokens())
758 return self.s[start:end]
759
760 def GetSpanForAttrValue(self, attr_name):
761 # type: (str) -> Tuple[int, int]
762 """
763 Used by oils_doc.py, for href shortcuts
764 """
765 # Algorithm: search for QuotedValue or UnquotedValue after AttrName
766 # TODO: Could also cache these
767
768 events = self.Tokens()
769 val = (-1, -1)
770 try:
771 while True:
772 tok_id, start, end = next(events)
773 if tok_id == h8_tag_id.AttrName:
774 name = self.s[start:end]
775 if name == attr_name:
776 # The value should come next
777 tok_id, start, end = next(events)
778 assert tok_id in (
779 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
780 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
781 val = start, end
782 break
783
784 except StopIteration:
785 pass
786 return val
787
788 def GetAttrRaw(self, attr_name):
789 # type: (str) -> Optional[str]
790 """
791 Return the value, which may be UNESCAPED.
792 """
793 start, end = self.GetSpanForAttrValue(attr_name)
794 if start == -1:
795 return None
796 return self.s[start:end]
797
798 def AllAttrsRawSlice(self):
799 # type: () -> List[Tuple[str, int, int]]
800 """
801 Get a list of pairs [('class', 3, 5), ('href', 9, 12)]
802 """
803 slices = []
804 events = self.Tokens()
805 try:
806 while True:
807 tok_id, start, end = next(events)
808 if tok_id == h8_tag_id.AttrName:
809 name = self.s[start:end]
810
811 # The value should come next
812 tok_id, start, end = next(events)
813 assert tok_id in (
814 h8_tag_id.QuotedValue, h8_tag_id.UnquotedValue,
815 h8_tag_id.MissingValue), h8_tag_id_str(tok_id)
816 # Note: quoted values may have &amp;
817 # We would need ANOTHER lexer to unescape them, but we
818 # don't need that for ul-table
819 slices.append((name, start, end))
820 except StopIteration:
821 pass
822 return slices
823
824 def AllAttrsRaw(self):
825 # type: () -> List[Tuple[str, str]]
826 """
827 Get a list of pairs [('class', 'foo'), ('href', '?foo=1&amp;bar=2')]
828
829 The quoted values may be escaped. We would need another lexer to
830 unescape them.
831 """
832 slices = self.AllAttrsRawSlice()
833 pairs = []
834 for name, start, end in slices:
835 pairs.append((name, self.s[start:end]))
836 return pairs
837
838 def Tokens(self):
839 # type: () -> Iterator[Tuple[h8_tag_id_t, int, int]]
840 """
841 Yields a sequence of tokens: Tag (AttrName AttrValue?)*
842
843 Where each Token is (Type, start_pos, end_pos)
844
845 Note that start and end are NOT redundant! We skip over some unwanted
846 characters.
847 """
848 m = _TAG_RE.match(self.s, self.start_pos + 1)
849 if not m:
850 raise RuntimeError("Couldn't find HTML tag in %r" %
851 self.WholeTagString())
852 yield h8_tag_id.TagName, m.start(1), m.end(1)
853
854 pos = m.end(0)
855 #log('POS %d', pos)
856
857 while True:
858 # don't search past the end
859 m = _ATTR_RE.match(self.s, pos, self.end_pos)
860 if not m:
861 #log('BREAK pos %d', pos)
862 break
863 #log('AttrName %r', m.group(1))
864
865 yield h8_tag_id.AttrName, m.start(1), m.end(1)
866
867 #log('m.groups() %r', m.groups())
868 if m.group(2) is not None:
869 # double quoted
870 yield h8_tag_id.QuotedValue, m.start(2), m.end(2)
871 elif m.group(3) is not None:
872 # single quoted - TODO: could have different token types
873 yield h8_tag_id.QuotedValue, m.start(3), m.end(3)
874 elif m.group(4) is not None:
875 yield h8_tag_id.UnquotedValue, m.start(4), m.end(4)
876 else:
877 # <button disabled>
878 end = m.end(0)
879 yield h8_tag_id.MissingValue, end, end
880
881 # Skip past the "
882 pos = m.end(0)
883
884 #log('TOK %r', self.s)
885
886 m = _TAG_LAST_RE.match(self.s, pos)
887 #log('_TAG_LAST_RE match %r', self.s[pos:])
888 if not m:
889 # Extra data at end of tag. TODO: add messages for all these.
890 raise LexError(self.s, pos)
891
892
893# This is similar but not identical to
894# " ([^>"\x00]*) " # double quoted value
895# | ' ([^>'\x00]*) ' # single quoted value
896#
897# Note: for unquoted values, & isn't allowed, and thus &amp; and &#99; and
898# &#x99; are not allowed. We could relax that?
899ATTR_VALUE_LEX = CHAR_LEX + [
900 (r'[^>&\x00]+', h8_id.RawData),
901 (r'.', h8_id.Invalid),
902]
903
904ATTR_VALUE_LEX_COMPILED = MakeLexer(ATTR_VALUE_LEX)
905
906
907class AttrValueLexer(object):
908 """
909 <a href="foo=99&amp;bar">
910 <a href='foo=99&amp;bar'>
911 <a href=unquoted>
912 """
913
914 def __init__(self, s):
915 # type: (str) -> None
916 self.s = s
917 self.start_pos = -1 # Invalid
918 self.end_pos = -1
919
920 def Reset(self, start_pos, end_pos):
921 # type: (int, int) -> None
922 """Reuse instances of this object."""
923 assert start_pos >= 0, start_pos
924 assert end_pos >= 0, end_pos
925
926 self.start_pos = start_pos
927 self.end_pos = end_pos
928
929 def NumTokens(self):
930 # type: () -> int
931 num_tokens = 0
932 pos = self.start_pos
933 for tok_id, end_pos in self.Tokens():
934 if tok_id == h8_id.Invalid:
935 raise LexError(self.s, pos)
936 pos = end_pos
937 #log('pos %d', pos)
938 num_tokens += 1
939 return num_tokens
940
941 def Tokens(self):
942 # type: () -> Iterator[Tuple[h8_id_t, int]]
943 pos = self.start_pos
944 while pos < self.end_pos:
945 # Find the first match, like above.
946 # Note: frontend/match.py uses _LongestMatch(), which is different!
947 # TODO: reconcile them. This lexer should be expressible in re2c.
948 for pat, tok_id in ATTR_VALUE_LEX_COMPILED:
949 m = pat.match(self.s, pos)
950 if m:
951 if 0:
952 tok_str = m.group(0)
953 log('token = %r', tok_str)
954
955 end_pos = m.end(0)
956 yield tok_id, end_pos
957 pos = end_pos
958 break
959 else:
960 raise AssertionError('h8_id.Invalid rule should have matched')