111 lines
2.1 KiB
Go
111 lines
2.1 KiB
Go
// Package lexer tokenises arithmetic expressions.
|
|
package lexer
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
)
|
|
|
|
type TokenType int
|
|
|
|
const (
|
|
NUMBER TokenType = iota
|
|
IDENT
|
|
PLUS
|
|
MINUS
|
|
STAR
|
|
SLASH
|
|
PERCENT
|
|
CARET // exponentiation
|
|
LPAREN
|
|
RPAREN
|
|
COMMA
|
|
ASSIGN
|
|
EOF
|
|
)
|
|
|
|
func (t TokenType) String() string {
|
|
return [...]string{"NUMBER", "IDENT", "+", "-", "*", "/", "%", "^", "(", ")", ",", "=", "EOF"}[t]
|
|
}
|
|
|
|
// Token carries its source column so errors can point at the input.
|
|
type Token struct {
|
|
Type TokenType
|
|
Value string
|
|
Column int
|
|
}
|
|
|
|
func (t Token) String() string { return fmt.Sprintf("%s(%q)@%d", t.Type, t.Value, t.Column) }
|
|
|
|
// Lex splits input into tokens.
|
|
func Lex(input string) ([]Token, error) {
|
|
var tokens []Token
|
|
i := 0
|
|
|
|
for i < len(input) {
|
|
c := input[i]
|
|
|
|
if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
|
|
i++
|
|
continue
|
|
}
|
|
|
|
if simple, ok := simpleToken(c); ok {
|
|
tokens = append(tokens, Token{Type: simple, Value: string(c), Column: i})
|
|
i++
|
|
continue
|
|
}
|
|
|
|
switch {
|
|
case isDigit(c):
|
|
start := i
|
|
for i < len(input) && isDigit(input[i]) {
|
|
i++
|
|
}
|
|
tokens = append(tokens, Token{Type: NUMBER, Value: input[start:i], Column: start})
|
|
|
|
case isAlpha(c):
|
|
start := i
|
|
for i < len(input) && (isAlpha(input[i]) || isDigit(input[i])) {
|
|
i++
|
|
}
|
|
tokens = append(tokens, Token{Type: IDENT, Value: strings.ToLower(input[start:i]), Column: start})
|
|
|
|
default:
|
|
return nil, fmt.Errorf("column %d: unexpected character %q", i, string(c))
|
|
}
|
|
}
|
|
|
|
tokens = append(tokens, Token{Type: EOF, Column: i})
|
|
return tokens, nil
|
|
}
|
|
|
|
func simpleToken(c byte) (TokenType, bool) {
|
|
switch c {
|
|
case '+':
|
|
return PLUS, true
|
|
case '-':
|
|
return MINUS, true
|
|
case '*':
|
|
return STAR, true
|
|
case '/':
|
|
return SLASH, true
|
|
case '%':
|
|
return PERCENT, true
|
|
case '^':
|
|
return CARET, true
|
|
case '(':
|
|
return LPAREN, true
|
|
case ')':
|
|
return RPAREN, true
|
|
case ',':
|
|
return COMMA, true
|
|
case '=':
|
|
return ASSIGN, true
|
|
}
|
|
return 0, false
|
|
}
|
|
|
|
func isDigit(c byte) bool { return c >= '0' && c <= '9' }
|
|
func isAlpha(c byte) bool { return c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') }
|