lox-go/scanner.go

package main

import (
	"fmt"
	"strconv"
)

var keywords = map[string]tokenType{
	"and":    tokenTypeAnd,
	"class":  tokenTypeClass,
	"else":   tokenTypeElse,
	"false":  tokenTypeFalse,
	"fun":    tokenTypeFun,
	"for":    tokenTypeFor,
	"if":     tokenTypeIf,
	"nil":    tokenTypeNil,
	"or":     tokenTypeOr,
	"print":  tokenTypePrint,
	"return": tokenTypeReturn,
	"super":  tokenTypeSuper,
	"this":   tokenTypeThis,
	"true":   tokenTypeTrue,
	"var":    tokenTypeVar,
	"while":  tokenTypeWhile,
}

func isDigit(r rune) bool {
	return r >= '0' && r <= '9'
}

func isAlpha(r rune) bool {
	return (r >= 'a' && r <= 'z') ||
		(r >= 'A' && r <= 'Z') ||
		r == '_'
}

func isAlphaNumeric(r rune) bool {
	return isDigit(r) || isAlpha(r)
}

type scanner struct {
	source []rune
	tokens []*token

	start   int
	current int
	line    int
}

func newScanner(source string) *scanner {
	s := new(scanner)

	s.source = []rune(source)
	s.tokens = []*token{}

	s.start = 0
	s.current = 0
	s.line = 1

	return s
}

func (s *scanner) isAtEnd() bool {
	return s.current >= len(s.source)
}

func (s *scanner) advance() rune {
	r := s.source[s.current]
	s.current += 1
	return r
}

func (s *scanner) addToken(t tokenType, literal any) {
	s.tokens = append(s.tokens, &token{
		Type:    t,
		Lexeme:  string(s.source[s.start:s.current]),
		Literal: literal,
		Line:    s.line,
	})
}

func (s *scanner) match(expected rune) bool {
	if s.isAtEnd() {
		return false
	}
	c := s.source[s.current]
	if c != expected {
		return false
	}

	s.current += 1
	return true
}

func (s *scanner) peek() rune {
	if s.isAtEnd() {
		return rune(0)
	}
	return s.source[s.current]
}

func (s *scanner) peekNext() rune {
	if s.current+1 > len(s.source) {
		return rune(0)
	}
	return s.source[s.current+1]
}

func (s *scanner) scanToken() bool {
	r := s.advance()

	switch r {
	// simple 1 character tokens
	case '(':
		s.addToken(tokenTypeLeftParen, nil)
	case ')':
		s.addToken(tokenTypeRightParen, nil)
	case '{':
		s.addToken(tokenTypeLeftBrace, nil)
	case '}':
		s.addToken(tokenTypeRightBrace, nil)
	case ',':
		s.addToken(tokenTypeComma, nil)
	case '.':
		s.addToken(tokenTypeDot, nil)
	case '-':
		s.addToken(tokenTypeMinus, nil)
	case '+':
		s.addToken(tokenTypePlus, nil)
	case ';':
		s.addToken(tokenTypeSemicolon, nil)
	case '*':
		s.addToken(tokenTypeStar, nil)

	// simple 2 character tokens
	case '!':
		if s.match('=') {
			s.addToken(tokenTypeBangEq, nil)
		} else {
			s.addToken(tokenTypeBang, nil)
		}
	case '=':
		if s.match('=') {
			s.addToken(tokenTypeEqualEqual, nil)
		} else {
			s.addToken(tokenTypeEqual, nil)
		}
	case '<':
		if s.match('=') {
			s.addToken(tokenTypeLessEq, nil)
		} else {
			s.addToken(tokenTypeLess, nil)
		}
	case '>':
		if s.match('=') {
			s.addToken(tokenTypeGreaterEq, nil)
		} else {
			s.addToken(tokenTypeGreater, nil)
		}

	case '/':
		// match comments
		if s.match('/') {
			// we scan until the end of line/file (whichever comes first :p)
			for s.peek() != '\n' && !s.isAtEnd() {
				s.advance()
			}
		} else {
			s.addToken(tokenTypeSlash, nil)
		}

	// ignore whitespace
	case ' ':
		break
	case '\r':
		break
	case '\t':
		break

	// advance the line counter :D
	case '\n':
		s.line += 1

	// string literals
	case '"':
		return s.string()

	default:
		if isDigit(r) {
			return s.number()
		} else if isAlpha(r) {
			s.identifier()
			return false
		}

		reportErr(s.line, fmt.Sprintf("Unexpected character %c", r))

		return true
	}

	return false
}

func (s *scanner) string() bool {
	// peek until we hit the end of the string or file, whichever is first
	for s.peek() != '"' && !s.isAtEnd() {
		// support strings with new lines :D
		if s.peek() == '\n' {
			s.line += 1
		}
		s.advance()
	}

	// if the token didn't end before the file we report and err
	// and return that we got one
	if s.isAtEnd() {
		reportErr(s.line, "Unterminated string")
		return true
	}

	s.advance()

	// todo: escape sequences
	value := s.source[s.start+1 : s.current-1]
	s.addToken(tokenTypeString, value)

	return false
}

func (s *scanner) number() bool {
	for isDigit(s.peek()) {
		s.advance()
	}

	if s.peek() == '.' && isDigit(s.peekNext()) {
		s.advance()

		for isDigit(s.peek()) {
			s.advance()
		}
	}

	literal, _ := strconv.ParseFloat(string(s.source[s.start:s.current]), 64)
	s.addToken(tokenTypeNumber, literal)

	return false
}

func (s *scanner) identifier() {
	for isAlphaNumeric(s.peek()) {
		s.advance()
	}

	text := s.source[s.start:s.current]
	tt, ok := keywords[string(text)]

	if !ok {
		tt = tokenTypeIdentifier
	}

	s.addToken(tt, nil)
}

func (s *scanner) ScanTokens() ([]*token, bool) {
	isErr := false

	for !s.isAtEnd() {
		s.start = s.current
		isErr = isErr || s.scanToken()
	}

	s.tokens = append(s.tokens, &token{
		Type:    tokenTypeEOF,
		Lexeme:  "",
		Literal: nil,
		Line:    s.line,
	})

	return s.tokens, !isErr
}