wrote dsl for ast boiler plate generation

ast/gen/lex.go

@@ -0,0 +1,180 @@
+package main
+
+import (
+	"errors"
+	"fmt"
+)
+
+type tokenType int
+
+const (
+	tokenTypeIdentifier tokenType = iota
+	tokenTypeRightBracket
+	tokenTypeLeftBracket
+	tokenTypeEqual
+	tokenTypeName
+	tokenTypeSemicolon
+	tokenTypeEOF
+)
+
+type token struct {
+	Type   tokenType
+	Lexeme string
+	Line   int
+}
+
+func isUpperAlpha(r rune) bool {
+	return r >= 'A' && r <= 'Z'
+}
+
+func isAlpha(r rune) bool {
+	return isUpperAlpha(r) || (r >= 'a' && r <= 'z')
+}
+
+func isNumeric(r rune) bool {
+	return r >= '0' && r <= '9'
+}
+
+func isAlphaNumeric(r rune) bool {
+	return isAlpha(r) || isNumeric(r)
+}
+
+func isGoIdentifier(r rune) bool {
+	return isAlphaNumeric(r) || r == '_'
+}
+
+func isIdentifier(r rune) bool {
+	return isGoIdentifier(r) || r == '.' || r == '*'
+}
+
+type lexer struct {
+	source  []rune
+	current int
+	start   int
+	line    int
+	tokens  []*token
+}
+
+func (l *lexer) addToken(t tokenType) {
+	l.tokens = append(l.tokens, &token{
+		Type:   t,
+		Lexeme: string(l.source[l.start:l.current]),
+		Line:   l.line,
+	})
+}
+
+func (l *lexer) peek() rune {
+	if l.isAtEnd() {
+		return rune(0)
+	}
+	return l.source[l.current]
+}
+
+func (l *lexer) peekNext() rune {
+	if l.current+1 >= len(l.source) {
+		return rune(0)
+	}
+	return l.source[l.current+1]
+}
+
+func (l *lexer) advance() rune {
+	r := l.source[l.current]
+	l.current += 1
+	return r
+}
+
+func (l *lexer) isAtEnd() bool {
+	return l.current >= len(l.source)
+}
+
+func (l *lexer) scanToken() error {
+	r := l.advance()
+
+	switch r {
+	case '[':
+		l.addToken(tokenTypeLeftBracket)
+	case ']':
+		l.addToken(tokenTypeRightBracket)
+	case '=':
+		l.addToken(tokenTypeEqual)
+	case ';':
+		l.addToken(tokenTypeSemicolon)
+	case ' ', '\r', '\t':
+		break
+	case '#':
+		next := l.peek()
+		if isUpperAlpha(next) {
+			l.name()
+			return nil
+		}
+		return errors.New(fmt.Sprintf("names must have an uppercase alphabetical first character, found '%s'", string(next)))
+	case '\n':
+		l.line += 1
+	default:
+		if isIdentifier(r) {
+			l.identifier()
+			return nil
+		}
+
+		return errors.New(fmt.Sprintf("unexpected character '%s' at line %d", string(r), l.line))
+	}
+
+	return nil
+}
+
+func (l *lexer) name() {
+	for isGoIdentifier(l.peek()) && l.peek() != '\n' {
+		l.advance()
+	}
+
+	text := l.source[l.start+1 : l.current]
+	l.advance()
+
+	l.tokens = append(l.tokens, &token{
+		Type:   tokenTypeName,
+		Lexeme: string(text),
+		Line:   l.line,
+	})
+}
+
+func (l *lexer) identifier() {
+	for isIdentifier(l.peek()) {
+		l.advance()
+	}
+
+	text := l.source[l.start:l.current]
+
+	l.tokens = append(l.tokens, &token{
+		Type:   tokenTypeIdentifier,
+		Lexeme: string(text),
+		Line:   l.line,
+	})
+}
+
+func (l *lexer) scanTokens() ([]*token, []error) {
+	errs := []error{}
+	for !l.isAtEnd() {
+		l.start = l.current
+		err := l.scanToken()
+		if err != nil {
+			errs = append(errs, err)
+		}
+	}
+
+	l.addToken(tokenTypeEOF)
+	return l.tokens, errs
+}
+
+func lex(source string) ([]*token, []error) {
+	l := new(lexer)
+
+	l.source = []rune(source)
+	l.current = 0
+	l.start = 0
+	l.line = 1
+	l.tokens = []*token{}
+
+	tokens, errs := l.scanTokens()
+
+	return tokens, errs
+}