package main import ( "fmt" "strconv" ) var keywords = map[string]tokenType{ "and": tokenTypeAnd, "class": tokenTypeClass, "else": tokenTypeElse, "false": tokenTypeFalse, "fun": tokenTypeFun, "for": tokenTypeFor, "if": tokenTypeIf, "nil": tokenTypeNil, "or": tokenTypeOr, "print": tokenTypePrint, "return": tokenTypeReturn, "super": tokenTypeSuper, "this": tokenTypeThis, "true": tokenTypeTrue, "var": tokenTypeVar, "while": tokenTypeWhile, } func isDigit(r rune) bool { return r >= '0' && r <= '9' } func isAlpha(r rune) bool { return (r >= 'a' && r <= 'z') || (r >= 'A' && r <= 'Z') || r == '_' } func isAlphaNumeric(r rune) bool { return isDigit(r) || isAlpha(r) } type scanner struct { source []rune tokens []*token start int current int line int } func newScanner(source string) *scanner { s := new(scanner) s.source = []rune(source) s.tokens = []*token{} s.start = 0 s.current = 0 s.line = 1 return s } func (s *scanner) isAtEnd() bool { return s.current >= len(s.source) } func (s *scanner) advance() rune { r := s.source[s.current] s.current += 1 return r } func (s *scanner) addToken(t tokenType, literal any) { s.tokens = append(s.tokens, &token{ Type: t, Lexeme: string(s.source[s.start:s.current]), Literal: literal, Line: s.line, }) } func (s *scanner) match(expected rune) bool { if s.isAtEnd() { return false } c := s.source[s.current] if c != expected { return false } s.current += 1 return true } func (s *scanner) peek() rune { if s.isAtEnd() { return rune(0) } return s.source[s.current] } func (s *scanner) peekNext() rune { if s.current+1 > len(s.source) { return rune(0) } return s.source[s.current+1] } func (s *scanner) scanToken() bool { r := s.advance() switch r { // simple 1 character tokens case '(': s.addToken(tokenTypeLeftParen, nil) case ')': s.addToken(tokenTypeRightParen, nil) case '{': s.addToken(tokenTypeLeftBrace, nil) case '}': s.addToken(tokenTypeRightBrace, nil) case ',': s.addToken(tokenTypeComma, nil) case '.': s.addToken(tokenTypeDot, nil) case '-': s.addToken(tokenTypeMinus, nil) case '+': s.addToken(tokenTypePlus, nil) case ';': s.addToken(tokenTypeSemicolon, nil) case '*': s.addToken(tokenTypeStar, nil) // simple 2 character tokens case '!': if s.match('=') { s.addToken(tokenTypeBangEq, nil) } else { s.addToken(tokenTypeBang, nil) } case '=': if s.match('=') { s.addToken(tokenTypeEqualEqual, nil) } else { s.addToken(tokenTypeEqual, nil) } case '<': if s.match('=') { s.addToken(tokenTypeLessEq, nil) } else { s.addToken(tokenTypeLess, nil) } case '>': if s.match('=') { s.addToken(tokenTypeGreaterEq, nil) } else { s.addToken(tokenTypeGreater, nil) } case '/': // match comments if s.match('/') { // we scan until the end of line/file (whichever comes first :p) for s.peek() != '\n' && !s.isAtEnd() { s.advance() } } else { s.addToken(tokenTypeSlash, nil) } // ignore whitespace case ' ': break case '\r': break case '\t': break // advance the line counter :D case '\n': s.line += 1 // string literals case '"': return s.string() default: if isDigit(r) { return s.number() } else if isAlpha(r) { s.identifier() return false } reportErr(s.line, fmt.Sprintf("Unexpected character %c", r)) return true } return false } func (s *scanner) string() bool { // peek until we hit the end of the string or file, whichever is first for s.peek() != '"' && !s.isAtEnd() { // support strings with new lines :D if s.peek() == '\n' { s.line += 1 } s.advance() } // if the token didn't end before the file we report and err // and return that we got one if s.isAtEnd() { reportErr(s.line, "Unterminated string") return true } s.advance() // todo: escape sequences value := s.source[s.start+1 : s.current-1] s.addToken(tokenTypeString, string(value)) return false } func (s *scanner) number() bool { for isDigit(s.peek()) { s.advance() } if s.peek() == '.' && isDigit(s.peekNext()) { s.advance() for isDigit(s.peek()) { s.advance() } } literal, _ := strconv.ParseFloat(string(s.source[s.start:s.current]), 64) s.addToken(tokenTypeNumber, literal) return false } func (s *scanner) identifier() { for isAlphaNumeric(s.peek()) { s.advance() } text := s.source[s.start:s.current] tt, ok := keywords[string(text)] if !ok { tt = tokenTypeIdentifier } s.addToken(tt, nil) } func (s *scanner) ScanTokens() ([]*token, bool) { isErr := false for !s.isAtEnd() { s.start = s.current isErr = isErr || s.scanToken() } s.tokens = append(s.tokens, &token{ Type: tokenTypeEOF, Lexeme: "", Literal: nil, Line: s.line, }) return s.tokens, !isErr }