NTERA/NTERA.Interpreter/Compiler/Lexer.cs

using System;
using System.Collections;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;

namespace NTERA.Interpreter.Compiler
{
	public class Lexer : IEnumerable<Token>
	{
		private readonly string source;
		private Marker sourceMarker;

		public char CurrentChar;

		private readonly IEnumerator<Token> currentEnumerator;

		public Marker TokenMarker { get; set; }

		public string Identifier { get; set; }
		public Value Value { get; set; }

		static Lexer()
		{
			InitTokenDictionaries();
		}

		public Lexer(string input)
		{
			source = input;
			sourceMarker = new Marker(-1, 1, 0);

			currentEnumerator = GetTokens();
			currentEnumerator.MoveNext();
		}

		public void GoTo(Marker marker)
		{
			sourceMarker = marker;
		}

		public bool IsPeeking { get; protected set; }

		public char GetNextChar(bool peek = false)
		{
			IsPeeking = peek;

			if (sourceMarker.Pointer + 1 >= source.Length)
			{
				sourceMarker.Pointer = source.Length;
				return CurrentChar = (char)0;
			}

			if (peek)
				return CurrentChar = source[sourceMarker.Pointer + 1];

			sourceMarker.Column++;
			sourceMarker.Pointer++;

			if ((CurrentChar = source[sourceMarker.Pointer]) == '\n')
			{
				sourceMarker.Column = 0;
				sourceMarker.Line++;
			}

			return CurrentChar;
		}

		protected static Dictionary<string, Token> TokenDictionary;

		protected static Dictionary<char, Token> TokenCharDictionary;

		private static bool _initialized = false;
		private static readonly object _initializedLock = new object();

		private static void InitTokenDictionaries()
		{
			if (_initialized)
				return;

			lock (_initializedLock)
			{
				if (_initialized)
					return;

				if (TokenDictionary == null)
				{
					TokenDictionary = new Dictionary<string, Token>(StringComparer.InvariantCultureIgnoreCase);

					foreach (Token token in Enum.GetValues(typeof(Token)))
					{
						foreach (var attribute in Utility.GetEnumAttributes<Token, LexerKeywordAttribute>(token))
						{
							TokenDictionary[attribute.Keyword] = token;
						}
					}
				}

				if (TokenCharDictionary == null)
				{
					TokenCharDictionary = new Dictionary<char, Token>();

					foreach (Token token in Enum.GetValues(typeof(Token)))
					{
						foreach (var attribute in Utility.GetEnumAttributes<Token, LexerCharacterAttribute>(token))
						{
							TokenCharDictionary[attribute.Character] = token;
						}
					}
				}
			}
		}

		private static Regex PowRegex = new Regex(@"(\d+)p(\d+)");

		private static bool IsWhitespace(char c)
		{
			return char.IsWhiteSpace(c) && c != '\n';
		}

		private static bool IsEndOfLine(char c)
		{
			return c == '\n' || c == '\r' || c == '\0';
		}

		private Token DetermineToken(bool peek, bool useCurrent)
		{
			char c = useCurrent ? CurrentChar : GetNextChar(peek);

			if (TokenCharDictionary.TryGetValue(c, out Token charToken))
				return charToken;

			switch (c)
			{
				case ';': //semicolon is comment
					while (CurrentChar != '\n')
					{
						if (CurrentChar == '\0')
							return Token.EOF;

						GetNextChar();
					}

					return Token.NewLine;

				case '[':
					const string SkipStart = "[SKIPSTART]";
					const string SkipEnd = "[SKIPEND]";

					if (sourceMarker.Column > 1
						|| source.Substring(sourceMarker.Pointer, SkipStart.Length) != SkipStart)
						return Token.Unknown;

					while (GetNextChar() != '\0')
					{
						if (CurrentChar == '[' && source.Substring(sourceMarker.Pointer, SkipEnd.Length) == SkipEnd)
						{
							while (true)
							{
								switch (GetNextChar())
								{
									case '\n':
										return Token.NewLine;
									case '\0':
										return Token.EOF;
								}
							}
						}
					}

					return Token.EOF;

				case '<':
					if (GetNextChar(true) == '>')
					{
						GetNextChar();
						return Token.NotEqual;
					}
					else if (GetNextChar(true) == '=')
					{
						GetNextChar();
						return Token.LessEqual;
					}
					else
						return Token.Less;

				case '>':
					if (GetNextChar(true) == '=')
					{
						GetNextChar();
						return Token.MoreEqual;
					}
					else
						return Token.More;

				case '+':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '+')
					{
						GetNextChar();
						return Token.Increment;
					}
					else
						return Token.Plus;

				case '-':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '-')
					{
						GetNextChar();
						return Token.Decrement;
					}
					else
						return Token.Minus;

				case '=':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '=')
						GetNextChar();

					return Token.Equal;

				case '&':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '&')
						GetNextChar();

					return Token.And;

				case '\\':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '@')
					{
						GetNextChar();
						return Token.TernaryEscape;
					}

					return Token.Unknown;

				case '|':
					if (peek)
						GetNextChar();

					if (GetNextChar(true) == '|')
						GetNextChar();

					return Token.Or;

				case (char)0:
					return Token.EOF;
			}

			return Token.Unknown;
		}

		private IEnumerator<Token> GetTokens()
		{
			sourceMarker = new Marker(-1, 1, 0);

			while (true)
			{
				while (IsWhitespace(GetNextChar()))
				{
				}

				TokenMarker = sourceMarker;

				Token token = DetermineToken(false, true);

				if (token == Token.EOF)
				{
					yield return Token.EOF;
					yield break;
				}

				if (token != Token.Unknown)
				{
					yield return token;
					continue;
				}

				StringBuilder bodyBuilder = new StringBuilder(CurrentChar.ToString());

				while (DetermineToken(true, false) == Token.Unknown
					   && !IsWhitespace(GetNextChar(true)))
				{
					bodyBuilder.Append(GetNextChar());
				}

				string result = bodyBuilder.ToString();

				if (double.TryParse(result, NumberStyles.Float, CultureInfo.InvariantCulture, out var real))
				{
					Value = real;
					yield return Token.Value;
					continue;
				}

				if (result.StartsWith("0x") && int.TryParse(result.Replace("0x", ""), NumberStyles.HexNumber, CultureInfo.CurrentCulture, out int hexResult))
				{
					Value = hexResult;
					yield return Token.Value;
					continue;
				}

				Match powMatch = PowRegex.Match(result);
				if (powMatch.Success)
				{
					int a = int.Parse(powMatch.Groups[1].Value);
					int b = int.Parse(powMatch.Groups[2].Value);

					Value = a << b;
					yield return Token.Value;
					continue;
				}

				Identifier = bodyBuilder.ToString();

				if (TokenDictionary.TryGetValue(Identifier, out token))
				{
					yield return token;
					continue;
				}

				yield return Token.Identifer;

				if (CurrentChar == '\n')
					yield return Token.NewLine;
			}
		}

		public IEnumerator<Token> GetEnumerator()
		{
			return currentEnumerator;
		}

		IEnumerator IEnumerable.GetEnumerator()
		{
			return GetEnumerator();
		}

		private static readonly Dictionary<Token, int> OrderOfOps = new Dictionary<Token, int>
		{
			{ Token.Or, 0 }, { Token.And, 0 },
			{ Token.Equal, 1 }, { Token.NotEqual, 1 },
			{ Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 }, { Token.MoreEqual, 1 },
			{ Token.Plus, 2 }, { Token.Minus, 2 },
			{ Token.Asterisk, 3 }, { Token.Slash, 3 },
			{ Token.Caret, 4 }
		};

		public Value Expression()
		{
			Stack<Value> stack = new Stack<Value>();
			Stack<Token> operators = new Stack<Token>();

			void Operation(Token token)
			{
				Value b = stack.Pop();
				Value a = stack.Pop();
				Value result = a.Operate(b, token);
				stack.Push(result);
			}

			int i = 0;
			while (true)
			{
				if (currentEnumerator.Current == Token.Value)
				{
					stack.Push(Value);
				}
				else if (currentEnumerator.Current == Token.Identifer)
				{
					throw new ParserException("Undeclared variable " + Identifier, TokenMarker);
				}
				else if (currentEnumerator.Current == Token.LParen)
				{
					currentEnumerator.MoveNext();
					stack.Push(Expression());

					if (currentEnumerator.Current != Token.RParen)
						throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);
				}
				else if (currentEnumerator.Current.IsArithmetic() && currentEnumerator.Current.IsUnary() && i == 0)
				{
					stack.Push(0);
					operators.Push(currentEnumerator.Current);
				}
				else if (currentEnumerator.Current.IsArithmetic())
				{
					while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()])
						Operation(operators.Pop());
					operators.Push(currentEnumerator.Current);
				}
				else
				{
					if (i == 0)
						throw new ParserException("Empty expression", TokenMarker);

					break;
				}

				i++;
				currentEnumerator.MoveNext();
			}

			while (operators.Count > 0)
				Operation(operators.Pop());

			return stack.Pop();
		}
	}
}