type ErrorHandler … type Scanner … const bom … const eof … // Read the next Unicode char into s.ch. // s.ch < 0 means end-of-file. // // For optimization, there is some overlap between this method and // s.scanIdentifier. func (s *Scanner) next() { … } // peek returns the byte following the most recently read character without // advancing the scanner. If the scanner is at EOF, peek returns 0. func (s *Scanner) peek() byte { … } type Mode … const ScanComments … const dontInsertSemis … // Init prepares the scanner s to tokenize the text src by setting the // scanner at the beginning of src. The scanner uses the file set file // for position information and it adds line information for each line. // It is ok to re-use the same file when re-scanning the same file as // line information which is already present is ignored. Init causes a // panic if the file size does not match the src size. // // Calls to [Scanner.Scan] will invoke the error handler err if they encounter a // syntax error and err is not nil. Also, for each error encountered, // the [Scanner] field ErrorCount is incremented by one. The mode parameter // determines how comments are handled. // // Note that Init may call err if there is an error in the first character // of the file. func (s *Scanner) Init(file *token.File, src []byte, err ErrorHandler, mode Mode) { … } func (s *Scanner) error(offs int, msg string) { … } func (s *Scanner) errorf(offs int, format string, args ...any) { … } // scanComment returns the text of the comment and (if nonzero) // the offset of the first newline within it, which implies a // /*...*/ comment. func (s *Scanner) scanComment() (string, int) { … } var prefix … // updateLineInfo parses the incoming comment text at offset offs // as a line directive. If successful, it updates the line info table // for the position next per the line directive. func (s *Scanner) updateLineInfo(next, offs int, text []byte) { … } func trailingDigits(text []byte) (int, int, bool) { … } func isLetter(ch rune) bool { … } func isDigit(ch rune) bool { … } // scanIdentifier reads the string of valid identifier characters at s.offset. // It must only be called when s.ch is known to be a valid letter. // // Be careful when making changes to this function: it is optimized and affects // scanning performance significantly. func (s *Scanner) scanIdentifier() string { … } func digitVal(ch rune) int { … } func lower(ch rune) rune { … } func isDecimal(ch rune) bool { … } func isHex(ch rune) bool { … } // digits accepts the sequence { digit | '_' }. // If base <= 10, digits accepts any decimal digit but records // the offset (relative to the source start) of a digit >= base // in *invalid, if *invalid < 0. // digits returns a bitset describing whether the sequence contained // digits (bit 0 is set), or separators '_' (bit 1 is set). func (s *Scanner) digits(base int, invalid *int) (digsep int) { … } func (s *Scanner) scanNumber() (token.Token, string) { … } func litname(prefix rune) string { … } // invalidSep returns the index of the first invalid separator in x, or -1. func invalidSep(x string) int { … } // scanEscape parses an escape sequence where rune is the accepted // escaped quote. In case of a syntax error, it stops at the offending // character (without consuming it) and returns false. Otherwise // it returns true. func (s *Scanner) scanEscape(quote rune) bool { … } func (s *Scanner) scanRune() string { … } func (s *Scanner) scanString() string { … } func stripCR(b []byte, comment bool) []byte { … } func (s *Scanner) scanRawString() string { … } func (s *Scanner) skipWhitespace() { … } func (s *Scanner) switch2(tok0, tok1 token.Token) token.Token { … } func (s *Scanner) switch3(tok0, tok1 token.Token, ch2 rune, tok2 token.Token) token.Token { … } func (s *Scanner) switch4(tok0, tok1 token.Token, ch2 rune, tok2, tok3 token.Token) token.Token { … } // Scan scans the next token and returns the token position, the token, // and its literal string if applicable. The source end is indicated by // [token.EOF]. // // If the returned token is a literal ([token.IDENT], [token.INT], [token.FLOAT], // [token.IMAG], [token.CHAR], [token.STRING]) or [token.COMMENT], the literal string // has the corresponding value. // // If the returned token is a keyword, the literal string is the keyword. // // If the returned token is [token.SEMICOLON], the corresponding // literal string is ";" if the semicolon was present in the source, // and "\n" if the semicolon was inserted because of a newline or // at EOF. // // If the returned token is [token.ILLEGAL], the literal string is the // offending character. // // In all other cases, Scan returns an empty literal string. // // For more tolerant parsing, Scan will return a valid token if // possible even if a syntax error was encountered. Thus, even // if the resulting token sequence contains no illegal tokens, // a client may not assume that no error occurred. Instead it // must check the scanner's ErrorCount or the number of calls // of the error handler, if there was one installed. // // Scan adds line information to the file added to the file // set with Init. Token positions are relative to that file // and thus relative to the file set. func (s *Scanner) Scan() (pos token.Pos, tok token.Token, lit string) { … }
Codebase Browser
go