NTERA/NTERA.Interpreter/Lexer.cs

using System;
using System.Collections.Generic;
using System.Text;

namespace NTERA.Interpreter
{
    public class Lexer
    {
        private readonly string source;
        private Marker sourceMarker;
        private char lastChar;

        public Marker TokenMarker { get; set; }

        public string Identifer { get; set; }
        public Value Value { get; set; }

        public Lexer(string input)
        {
            source = input;
            sourceMarker = new Marker(-1, 1, 0);
        }

        public void GoTo(Marker marker)
        {
            sourceMarker = marker;
        }

        char GetChar()
        {
            sourceMarker.Column++;
            sourceMarker.Pointer++;

            if (sourceMarker.Pointer >= source.Length)
                return lastChar = (char)0;

            if ((lastChar = source[sourceMarker.Pointer]) == '\n')
            {
                sourceMarker.Column = 1;
                sourceMarker.Line++;
            }
            return lastChar;
        }

        private readonly Dictionary<string, Token> TokenDictionary = new Dictionary<string, Token>(StringComparer.InvariantCultureIgnoreCase)
        {
            //["PRINT"] = Token.Print,
            //["PRINTL"] = Token.PrintL,
            ["DRAWLINE"] = Token.DrawLine,
            //["DRAWLINEFORM"] = Token.DrawLineForm,
            ["DIM"] = Token.Dim,
            ["CONST"] = Token.Const,
            ["IF"] = Token.If,
            ["ENDIF"] = Token.EndIf,
            ["THEN"] = Token.Then,
            ["ELSE"] = Token.Else,
            ["FOR"] = Token.For,
            ["TO"] = Token.To,
            ["NEXT"] = Token.Next,
            ["INPUT"] = Token.Input,
            ["LET"] = Token.Let,
            ["GOSUB"] = Token.Gosub,
            ["RETURN"] = Token.Return,
            ["END"] = Token.End,
            ["OR"] = Token.Or,
            ["AND"] = Token.And,

            ["TIMES"] = Token.Times,
        };

        public IEnumerable<Token> GetTokens()
        {
            while (true)
            {
                GetChar();

                while (lastChar == ' ' || lastChar == '\t' || lastChar == '\r')
                    GetChar();

                TokenMarker = sourceMarker;

                if (char.IsLetter(lastChar))
                {
                    Identifer = lastChar.ToString();
                    while (char.IsLetterOrDigit(GetChar()) || lastChar == '_')
                        Identifer += lastChar;

                    if (TokenDictionary.ContainsKey(Identifer))
                    {
                        yield return TokenDictionary[Identifer];
                        continue;
                    }

                    switch (Identifer.ToUpper())
                    {
                        case "DRAWLINEFORM":
                            foreach (Token t in ReturnAsLine(Token.DrawLineForm))
                                yield return t;
                            continue;

                        case "PRINTFORML":
                            foreach (Token t in ReturnAsLine(Token.PrintFormL))
                                yield return t;
                            continue;

                        case "PRINT":
                            foreach (Token t in ReturnAsLine(Token.Print))
                                yield return t;
                            continue;

                        case "REM":
                            while (lastChar != '\n')
                                GetChar();

                            continue;
                        default:
                        {
                            yield return Token.Identifer;

                            sourceMarker.Pointer--;

                            continue;
                        }
                    }
                }

                if (char.IsDigit(lastChar))
                {
                    string num = "";

                    do
                    {
                        num += lastChar;
                    }
                    while (char.IsDigit(GetChar()) || lastChar == '.');

                    if (!double.TryParse(num, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var real))
                        throw new Exception("ERROR while parsing number");

                    Value = new Value(real);
                    yield return Token.Value;
                    
                    sourceMarker.Pointer--;

                    continue;
                }
                
                switch (lastChar)
                {
                    case '\n': yield return Token.NewLine; continue;
                    case '@': yield return Token.Function; continue;
                    case '#': yield return Token.Global; continue;
                    case ':': yield return Token.Colon; continue;
                    case ';': yield return Token.Semicolon; continue;
                    case ',': yield return Token.Comma; continue;
                    case '=': yield return Token.Equal; continue;
                    case '+': yield return Token.Plus; continue;
                    case '-': yield return Token.Minus; continue;
                    case '/': yield return Token.Slash; continue;
                    case '*': yield return Token.Asterisk; continue;
                    case '^': yield return Token.Caret; continue;
                    case '(': yield return Token.LParen; continue;
                    case ')': yield return Token.RParen; continue;
                    case '\'':
                        while (lastChar != '\n')
                            GetChar();
                        continue;
                    case '<':
                        GetChar();
                        if (lastChar == '>')
                            yield return Token.NotEqual;
                        else if (lastChar == '=')
                            yield return Token.LessEqual;
                        else
                            yield return Token.Less;
                        continue;
                    case '>':
                        GetChar();
                        if (lastChar == '=')
                            yield return Token.MoreEqual;
                        else
                            yield return Token.More;
                        continue;
                    case '"':
                        string str = "";
                        while (GetChar() != '"')
                        {
                            if (lastChar == '\\')
                            {
                                switch (char.ToLower(GetChar()))
                                {
                                    case 'n': str += '\n'; break;
                                    case 't': str += '\t'; break;
                                    case '\\': str += '\\'; break;
                                    case '"': str += '"'; break;
                                }
                            }
                            else
                            {
                                str += lastChar;
                            }
                        }
                        Value = new Value(str);
                        yield return Token.Value;
                        continue;
                    case (char)0:
                        yield return Token.EOF;
                        break;
                    default:
                        yield return Token.Unkown;
                        continue;
                }

                break;
            }
        }

        public IEnumerable<Token> ReturnAsLine(Token token)
        {
            StringBuilder bodyBuilder = new StringBuilder();

            while (lastChar != '\n')
                bodyBuilder.Append(GetChar());

            yield return token;

            Value = new Value(bodyBuilder.ToString());
            yield return Token.Value;

            yield return Token.NewLine;
        }
    }
}