NTERA/NTERA.Interpreter/Lexer.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;

namespace NTERA.Interpreter
{
    public class Lexer : IEnumerable<Token>
    {
        private readonly string source;
        private Marker sourceMarker;
        private char currentChar;

        private IEnumerator<Token> currentEnumerator;

        private LexerType _type;
        public LexerType Type
        {
            get => _type;
            internal set
            {
                _type = value;
                InitTokenDictionaries();
            }
        }

        public Marker TokenMarker { get; set; }

        public string Identifer { get; set; }
        public Value Value { get; set; }

        public Lexer(string input, LexerType type = LexerType.Both)
        {
            Type = type;

            source = input;
            sourceMarker = new Marker(-1, 1, 0);

            currentEnumerator = GetTokens();
            currentEnumerator.MoveNext();
        }

        public void GoTo(Marker marker)
        {
            sourceMarker = marker;
        }

        char GetNextChar(bool peek = false)
        {
            if (sourceMarker.Pointer + 1 >= source.Length)
            {
                sourceMarker.Pointer = source.Length;
                return currentChar = (char)0;
            }

            if (peek)
                return currentChar = source[sourceMarker.Pointer + 1];

            sourceMarker.Column++;
            sourceMarker.Pointer++;

            if ((currentChar = source[sourceMarker.Pointer]) == '\n')
            {
                sourceMarker.Column = 1;
                sourceMarker.Line++;
            }
            return currentChar;
        }

        private Dictionary<string, Token> TokenDictionary;

        private Dictionary<string, Token> TokenLineDictionary;

        private Dictionary<char, Token> TokenCharDictionary;

        private void InitTokenDictionaries()
        {
            TokenDictionary = new Dictionary<string, Token>(StringComparer.InvariantCultureIgnoreCase);
            TokenLineDictionary = new Dictionary<string, Token>(StringComparer.InvariantCultureIgnoreCase);
            TokenCharDictionary = new Dictionary<char, Token>();

            foreach (Token token in Enum.GetValues(typeof(Token)))
            {
                foreach (var attribute in Utility.GetEnumAttributes<Token, LexerKeywordAttribute>(token))
                {
                    if (attribute.IsLineKeyword)
                        TokenLineDictionary[attribute.Keyword] = token;
                    else
                        TokenDictionary[attribute.Keyword] = token;
                }

                foreach (var attribute in Utility.GetEnumAttributes<Token, LexerCharacterAttribute>(token))
                {
                    if ((attribute.LexerContext & Type) > 0)
                        TokenCharDictionary[attribute.Character] = token;
                }
            }
        }

        private static bool IsWhitespace(char c)
        {
            return char.IsWhiteSpace(c) && c != '\n';
        }

        private static bool IsEndOfLine(char c)
        {
            return c == '\n' || c == '\r' || c == '\0';
        }

        private static bool IsEscape(char c)
        {
            return c == '%' || c == '{';
        }

        private Token DetermineToken(char c)
        {
            if (TokenCharDictionary.TryGetValue(currentChar, out Token charToken))
                return charToken;

            switch (currentChar)
            {
                case ';': //semicolon is comment
                    while (currentChar != '\n')
                        GetNextChar();

                    return Token.NewLine;
                case '<':
                    if (!Type.HasFlag(LexerType.Real))
                        break;

                    if (GetNextChar(true) == '>')
                    {
                        GetNextChar();
                        return Token.NotEqual;
                    }
                    else if (GetNextChar(true) == '=')
                    {
                        GetNextChar();
                        return Token.LessEqual;
                    }
                    else
                        return Token.Less;

                case '>':
                    if (!Type.HasFlag(LexerType.Real))
                        break;

                    if (GetNextChar(true) == '=')
                    {
                        GetNextChar();
                        return Token.MoreEqual;
                    }
                    else
                        return Token.More;

                case '+':
                    if (GetNextChar(true) == '=')
                    {
                        GetNextChar();
                        return Token.Append;
                    }
                    else
                        return Token.Plus;

                case '%':

                    StringBuilder builder = new StringBuilder();
                    while (GetNextChar() != '%')
                        builder.Append(currentChar);

                    Value = $"%{builder}%";
                    return Token.Value;

                case '"':
                    string str = "";
                    while (GetNextChar() != '"')
                    {
                        if (currentChar == '\\')
                        {
                            switch (char.ToLower(GetNextChar()))
                            {
                                case 'n': str += '\n'; break;
                                case 't': str += '\t'; break;
                                case '\\': str += '\\'; break;
                                case '"': str += '"'; break;
                            }
                        }
                        else
                        {
                            str += currentChar;
                        }
                    }
                    Value = new Value(str);
                    return Token.Value;

                case (char)0:
                    return Token.EOF;
            }

            return Token.Unknown;
        }

        private IEnumerator<Token> GetTokens()
        {
            sourceMarker = new Marker(-1, 1, 0);

            while (true)
            {
                while (IsWhitespace(GetNextChar()) && Type != LexerType.String || currentChar == '\r') { }

                TokenMarker = sourceMarker;

                Token token = DetermineToken(currentChar);

                if (token == Token.EOF)
                {
                    yield return Token.EOF;
                    yield break;
                }

                if (token != Token.Unknown)
                {
                    yield return token;
                    continue;
                }

                StringBuilder bodyBuilder = new StringBuilder(currentChar.ToString());

                while ((!IsEscape(GetNextChar(true)) || Type != LexerType.String)
                        && DetermineToken(GetNextChar(true)) == Token.Unknown
                        && (!IsWhitespace(GetNextChar(true)) || Type == LexerType.String)
                        && GetNextChar(true) != '\r')
                {
                    bodyBuilder.Append(GetNextChar());
                }

                string result = bodyBuilder.ToString();

                if (double.TryParse(result, NumberStyles.Float, CultureInfo.InvariantCulture, out var real))
                {
                    Value = real;
                    yield return Token.Value;
                    continue;
                }

                if (result.StartsWith("0x") && int.TryParse(result.Replace("0x", ""), NumberStyles.HexNumber, CultureInfo.CurrentCulture, out int hexResult))
                {
                    Value = hexResult;
                    yield return Token.Value;
                    continue;
                }

                Identifer = bodyBuilder.ToString();

                if (TokenDictionary.TryGetValue(Identifer, out token))
                {
                    yield return token;
                    continue;
                }

                if (Type == LexerType.String && char.IsWhiteSpace(Identifer[0]))
                    Identifer = Identifer.Substring(1);

                if (TokenLineDictionary.TryGetValue(Identifer, out token))
                {
                    bodyBuilder = new StringBuilder();

                    while (!IsEndOfLine(GetNextChar(true)))
                        bodyBuilder.Append(GetNextChar());
                    
                    yield return token;

                    string strValue = bodyBuilder.ToString();

                    if (strValue.Length > 0 && char.IsWhiteSpace(strValue[0]))
                        strValue = strValue.Substring(1);

                    Value = new Value(strValue);
                    yield return Token.Value;

                    yield return currentChar == '\0' ? Token.EOF : Token.NewLine;

                    continue;
                }

                yield return Token.Identifer;
            }
        }

        public IEnumerator<Token> GetEnumerator()
        {
            return currentEnumerator;
        }

        IEnumerator IEnumerable.GetEnumerator()
        {
            return GetEnumerator();
        }

        private static readonly Dictionary<Token, int> OrderOfOps = new Dictionary<Token, int>
        {
            { Token.Or, 0 }, { Token.And, 0 },
            { Token.Equal, 1 }, { Token.NotEqual, 1 },
            { Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 },  { Token.MoreEqual, 1 },
            { Token.Plus, 2 }, { Token.Minus, 2 },
            { Token.Asterisk, 3 }, {Token.Slash, 3 },
            { Token.Caret, 4 }
        };

        public Value Expression(Interpreter context = null)
        {
            Stack<Value> stack = new Stack<Value>();
            Stack<Token> operators = new Stack<Token>();

            void Operation(Token token)
            {
                Value b = stack.Pop();
                Value a = stack.Pop();
                Value result = a.Operate(b, token);
                stack.Push(result);
            }

            int i = 0;
            while (true)
            {
                if (currentEnumerator.Current == Token.Value)
                {
                    stack.Push(Value);
                }
                else if (currentEnumerator.Current == Token.Identifer)
                {
                    if (context != null)
                    {
                        if (context.Variables.ContainsKey(Identifer))
                        {
                            string varName = Identifer;
                            int index = 0;
                            currentEnumerator.MoveNext();
                            if (currentEnumerator.Current == Token.Colon)
                            {
                                currentEnumerator.MoveNext();
                                index = (int)Expression(context).Real;
                            }

                            stack.Push(context.Variables[varName, index]);

                            i++;
                            continue;
                        }

                        if (context.FunctionDictionary.ContainsKey(Identifer))
                        {
                            string name = Identifer;
                            List<Value> args = new List<Value>();
                            currentEnumerator.MoveNext();

                            if (currentEnumerator.Current != Token.LParen)
                                throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);

                            while (currentEnumerator.MoveNext() && currentEnumerator.Current != Token.RParen)
                            {
                                args.Add(Expression(context));
                                if (currentEnumerator.Current != Token.Comma)
                                    break;
                            }

                            stack.Push(context.FunctionDictionary[name](args));
                            currentEnumerator.MoveNext();
                            i++;
                            continue;
                        }
                    }
                    
                    if (Type == LexerType.String)
                        stack.Push(Identifer);
                    else
                        throw new ParserException("Undeclared variable " + Identifer, TokenMarker);
                }
                else if (currentEnumerator.Current == Token.LParen)
                {
                    currentEnumerator.MoveNext();
                    stack.Push(Expression());

                    if (currentEnumerator.Current != Token.RParen)
                        throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);
                }
                else if (Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic()
                         && currentEnumerator.Current.IsUnary() && (i == 0)) // || previousToken == Token.LParen))
                {
                    stack.Push(0);
                    operators.Push(currentEnumerator.Current);
                }
                else if (Type == LexerType.String && currentEnumerator.Current.IsStringOp()
                        || Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic())
                {
                    while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()])
                        Operation(operators.Pop());
                    operators.Push(currentEnumerator.Current);
                }
                else
                {
                    if (i == 0)
                    {
                        if (Type == LexerType.String)
                            stack.Push("");
                        else
                            throw new ParserException("Empty expression", TokenMarker);
                    }

                    break;
                }

                i++;
                currentEnumerator.MoveNext();
            }

            while (operators.Count > 0)
                Operation(operators.Pop());

            return Type == LexerType.String
                ? stack.Aggregate((a, b) => b.String + a.String)
                : stack.Pop();
        }
    }
}