type TokenType … const ErrorToken … const TextToken … const StartTagToken … const EndTagToken … const SelfClosingTagToken … const CommentToken … const DoctypeToken … var ErrBufferExceeded … // String returns a string representation of the TokenType. func (t TokenType) String() string { … } type Attribute … type Token … // tagString returns a string representation of a tag Token's Data and Attr. func (t Token) tagString() string { … } // String returns a string representation of the Token. func (t Token) String() string { … } type span … type Tokenizer … // AllowCDATA sets whether or not the tokenizer recognizes <![CDATA[foo]]> as // the text "foo". The default value is false, which means to recognize it as // a bogus comment "<!-- [CDATA[foo]] -->" instead. // // Strictly speaking, an HTML5 compliant tokenizer should allow CDATA if and // only if tokenizing foreign content, such as MathML and SVG. However, // tracking foreign-contentness is difficult to do purely in the tokenizer, // as opposed to the parser, due to HTML integration points: an <svg> element // can contain a <foreignObject> that is foreign-to-SVG but not foreign-to- // HTML. For strict compliance with the HTML5 tokenization algorithm, it is the // responsibility of the user of a tokenizer to call AllowCDATA as appropriate. // In practice, if using the tokenizer without caring whether MathML or SVG // CDATA is text or comments, such as tokenizing HTML to find all the anchor // text, it is acceptable to ignore this responsibility. func (z *Tokenizer) AllowCDATA(allowCDATA bool) { … } // NextIsNotRawText instructs the tokenizer that the next token should not be // considered as 'raw text'. Some elements, such as script and title elements, // normally require the next token after the opening tag to be 'raw text' that // has no child elements. For example, tokenizing "<title>a<b>c</b>d</title>" // yields a start tag token for "<title>", a text token for "a<b>c</b>d", and // an end tag token for "</title>". There are no distinct start tag or end tag // tokens for the "<b>" and "</b>". // // This tokenizer implementation will generally look for raw text at the right // times. Strictly speaking, an HTML5 compliant tokenizer should not look for // raw text if in foreign content: <title> generally needs raw text, but a // <title> inside an <svg> does not. Another example is that a <textarea> // generally needs raw text, but a <textarea> is not allowed as an immediate // child of a <select>; in normal parsing, a <textarea> implies </select>, but // one cannot close the implicit element when parsing a <select>'s InnerHTML. // Similarly to AllowCDATA, tracking the correct moment to override raw-text- // ness is difficult to do purely in the tokenizer, as opposed to the parser. // For strict compliance with the HTML5 tokenization algorithm, it is the // responsibility of the user of a tokenizer to call NextIsNotRawText as // appropriate. In practice, like AllowCDATA, it is acceptable to ignore this // responsibility for basic usage. // // Note that this 'raw text' concept is different from the one offered by the // Tokenizer.Raw method. func (z *Tokenizer) NextIsNotRawText() { … } // Err returns the error associated with the most recent ErrorToken token. // This is typically io.EOF, meaning the end of tokenization. func (z *Tokenizer) Err() error { … } // readByte returns the next byte from the input stream, doing a buffered read // from z.r into z.buf if necessary. z.buf[z.raw.start:z.raw.end] remains a contiguous byte // slice that holds all the bytes read so far for the current token. // It sets z.err if the underlying reader returns an error. // Pre-condition: z.err == nil. func (z *Tokenizer) readByte() byte { … } // Buffered returns a slice containing data buffered but not yet tokenized. func (z *Tokenizer) Buffered() []byte { … } // readAtLeastOneByte wraps an io.Reader so that reading cannot return (0, nil). // It returns io.ErrNoProgress if the underlying r.Read method returns (0, nil) // too many times in succession. func readAtLeastOneByte(r io.Reader, b []byte) (int, error) { … } // skipWhiteSpace skips past any white space. func (z *Tokenizer) skipWhiteSpace() { … } // readRawOrRCDATA reads until the next "</foo>", where "foo" is z.rawTag and // is typically something like "script" or "textarea". func (z *Tokenizer) readRawOrRCDATA() { … } // readRawEndTag attempts to read a tag like "</foo>", where "foo" is z.rawTag. // If it succeeds, it backs up the input position to reconsume the tag and // returns true. Otherwise it returns false. The opening "</" has already been // consumed. func (z *Tokenizer) readRawEndTag() bool { … } // readScript reads until the next </script> tag, following the byzantine // rules for escaping/hiding the closing tag. func (z *Tokenizer) readScript() { … } // readComment reads the next comment token starting with "<!--". The opening // "<!--" has already been consumed. func (z *Tokenizer) readComment() { … } func (z *Tokenizer) calculateAbruptCommentDataEnd() int { … } func hasSuffix(b []byte, suffix string) bool { … } // readUntilCloseAngle reads until the next ">". func (z *Tokenizer) readUntilCloseAngle() { … } // readMarkupDeclaration reads the next token starting with "<!". It might be // a "<!--comment-->", a "<!DOCTYPE foo>", a "<![CDATA[section]]>" or // "<!a bogus comment". The opening "<!" has already been consumed. func (z *Tokenizer) readMarkupDeclaration() TokenType { … } // readDoctype attempts to read a doctype declaration and returns true if // successful. The opening "<!" has already been consumed. func (z *Tokenizer) readDoctype() bool { … } // readCDATA attempts to read a CDATA section and returns true if // successful. The opening "<!" has already been consumed. func (z *Tokenizer) readCDATA() bool { … } // startTagIn returns whether the start tag in z.buf[z.data.start:z.data.end] // case-insensitively matches any element of ss. func (z *Tokenizer) startTagIn(ss ...string) bool { … } // readStartTag reads the next start tag token. The opening "<a" has already // been consumed, where 'a' means anything in [A-Za-z]. func (z *Tokenizer) readStartTag() TokenType { … } // readTag reads the next tag token and its attributes. If saveAttr, those // attributes are saved in z.attr, otherwise z.attr is set to an empty slice. // The opening "<a" or "</a" has already been consumed, where 'a' means anything // in [A-Za-z]. func (z *Tokenizer) readTag(saveAttr bool) { … } // readTagName sets z.data to the "div" in "<div k=v>". The reader (z.raw.end) // is positioned such that the first byte of the tag name (the "d" in "<div") // has already been consumed. func (z *Tokenizer) readTagName() { … } // readTagAttrKey sets z.pendingAttr[0] to the "k" in "<div k=v>". // Precondition: z.err == nil. func (z *Tokenizer) readTagAttrKey() { … } // readTagAttrVal sets z.pendingAttr[1] to the "v" in "<div k=v>". func (z *Tokenizer) readTagAttrVal() { … } // Next scans the next token and returns its type. func (z *Tokenizer) Next() TokenType { … } // Raw returns the unmodified text of the current token. Calling Next, Token, // Text, TagName or TagAttr may change the contents of the returned slice. // // The token stream's raw bytes partition the byte stream (up until an // ErrorToken). There are no overlaps or gaps between two consecutive token's // raw bytes. One implication is that the byte offset of the current token is // the sum of the lengths of all previous tokens' raw bytes. func (z *Tokenizer) Raw() []byte { … } // convertNewlines converts "\r" and "\r\n" in s to "\n". // The conversion happens in place, but the resulting slice may be shorter. func convertNewlines(s []byte) []byte { … } var nul … var replacement … // Text returns the unescaped text of a text, comment or doctype token. The // contents of the returned slice may change on the next call to Next. func (z *Tokenizer) Text() []byte { … } // TagName returns the lower-cased name of a tag token (the `img` out of // `<IMG SRC="foo">`) and whether the tag has attributes. // The contents of the returned slice may change on the next call to Next. func (z *Tokenizer) TagName() (name []byte, hasAttr bool) { … } // TagAttr returns the lower-cased key and unescaped value of the next unparsed // attribute for the current tag token and whether there are more attributes. // The contents of the returned slices may change on the next call to Next. func (z *Tokenizer) TagAttr() (key, val []byte, moreAttr bool) { … } // Token returns the current Token. The result's Data and Attr values remain // valid after subsequent Next calls. func (z *Tokenizer) Token() Token { … } // SetMaxBuf sets a limit on the amount of data buffered during tokenization. // A value of 0 means unlimited. func (z *Tokenizer) SetMaxBuf(n int) { … } // NewTokenizer returns a new HTML Tokenizer for the given Reader. // The input is assumed to be UTF-8 encoded. func NewTokenizer(r io.Reader) *Tokenizer { … } // NewTokenizerFragment returns a new HTML Tokenizer for the given Reader, for // tokenizing an existing element's InnerHTML fragment. contextTag is that // element's tag, such as "div" or "iframe". // // For example, how the InnerHTML "a<b" is tokenized depends on whether it is // for a <p> tag or a <script> tag. // // The input is assumed to be UTF-8 encoded. func NewTokenizerFragment(r io.Reader, contextTag string) *Tokenizer { … }
Codebase Browser
kubernetes