Clean up now useless string mode code

NTERA.Interpreter/Attributes.cs

@@ -2,25 +2,14 @@
 
 namespace NTERA.Interpreter
 {
-	[Flags]
-	public enum LexerType
-	{
-		Real = 1,
-		String = 2,
-		Both = Real | String
-	}
-
 	[AttributeUsage(AttributeTargets.Field, AllowMultiple = true)]
 	public class LexerCharacterAttribute : Attribute
 	{
 		public char Character { get; }
-		public LexerType LexerContext { get; }
 
-		public LexerCharacterAttribute(char character, LexerType lexerContext = LexerType.Both)
+		public LexerCharacterAttribute(char character)
 		{
 			Character = character;
-
-			LexerContext = lexerContext;
 		}
 	}
 

NTERA.Interpreter/Compiler/Lexer.cs

@@ -2,7 +2,6 @@ using System;
 using System.Collections;
 using System.Collections.Generic;
 using System.Globalization;
-using System.Linq;
 using System.Text;
 using System.Text.RegularExpressions;
 
@@ -17,27 +16,18 @@ namespace NTERA.Interpreter.Compiler
 
 		private readonly IEnumerator<Token> currentEnumerator;
 
-		private LexerType _type;
-
-		public LexerType Type
-		{
-			get => _type;
-			internal set
-			{
-				_type = value;
-				InitTokenDictionaries();
-			}
-		}
-
 		public Marker TokenMarker { get; set; }
 
 		public string Identifier { get; set; }
 		public Value Value { get; set; }
 
-		public Lexer(string input, LexerType type = LexerType.Both)
+		static Lexer()
 		{
-			Type = type;
+			InitTokenDictionaries();
+		}
 
+		public Lexer(string input)
+		{
 			source = input;
 			sourceMarker = new Marker(-1, 1, 0);
 
@@ -79,15 +69,12 @@ namespace NTERA.Interpreter.Compiler
 
 		protected static Dictionary<string, Token> TokenDictionary;
 
-		protected Dictionary<char, Token> TokenCharDictionary;
-
-		protected static Dictionary<char, Token> BothModeTokens;
-		protected static Dictionary<char, Token> StringModeTokens;
+		protected static Dictionary<char, Token> TokenCharDictionary;
 
 		private static bool _initialized = false;
 		private static readonly object _initializedLock = new object();
 
-		private void InitTokenDictionaries()
+		private static void InitTokenDictionaries()
 		{
 			if (_initialized)
 				return;
@@ -110,24 +97,18 @@ namespace NTERA.Interpreter.Compiler
 					}
 				}
 
-				if (BothModeTokens == null || StringModeTokens == null)
+				if (TokenCharDictionary == null)
 				{
-					BothModeTokens = new Dictionary<char, Token>();
-					StringModeTokens = new Dictionary<char, Token>();
+					TokenCharDictionary = new Dictionary<char, Token>();
 
 					foreach (Token token in Enum.GetValues(typeof(Token)))
 					{
 						foreach (var attribute in Utility.GetEnumAttributes<Token, LexerCharacterAttribute>(token))
 						{
-							if ((attribute.LexerContext & LexerType.String) > 0)
-								StringModeTokens[attribute.Character] = token;
-
-							BothModeTokens[attribute.Character] = token;
+							TokenCharDictionary[attribute.Character] = token;
 						}
 					}
 				}
-
-				TokenCharDictionary = Type == LexerType.String ? StringModeTokens : BothModeTokens;
 			}
 		}
 
@@ -190,13 +171,7 @@ namespace NTERA.Interpreter.Compiler
 
 					return Token.EOF;
 
-				case '%':
-					return Type == LexerType.String ? Token.Format : Token.Modulo;
-
 				case '<':
-					if (!Type.HasFlag(LexerType.Real))
-						break;
-
 					if (GetNextChar(true) == '>')
 					{
 						GetNextChar();
@@ -211,9 +186,6 @@ namespace NTERA.Interpreter.Compiler
 						return Token.Less;
 
 				case '>':
-					if (!Type.HasFlag(LexerType.Real))
-						break;
-
 					if (GetNextChar(true) == '=')
 					{
 						GetNextChar();
@@ -223,9 +195,6 @@ namespace NTERA.Interpreter.Compiler
 						return Token.More;
 
 				case '+':
-					if (Type == LexerType.String)
-						return Token.Unknown;
-
 					if (peek)
 						GetNextChar();
 
@@ -238,9 +207,6 @@ namespace NTERA.Interpreter.Compiler
 						return Token.Plus;
 
 				case '-':
-					if (Type == LexerType.String)
-						return Token.Unknown;
-
 					if (peek)
 						GetNextChar();
 
@@ -253,9 +219,6 @@ namespace NTERA.Interpreter.Compiler
 						return Token.Minus;
 
 				case '=':
-					if (Type == LexerType.String)
-						return Token.Unknown;
-
 					if (peek)
 						GetNextChar();
 
@@ -282,56 +245,6 @@ namespace NTERA.Interpreter.Compiler
 
 					return Token.Or;
 
-				case '@':
-					if (Type == LexerType.String)
-						return Token.Unknown;
-
-					if (GetNextChar(true) == '"')
-					{
-						GetNextChar();
-						goto case '"';
-					}
-
-					return Token.AtSymbol;
-
-				case '"':
-
-					//if (peek)
-					//	GetNextChar();
-
-					string str = "";
-
-					while (GetNextChar() != '"')
-					{
-						if (CurrentChar == '\\')
-						{
-							switch (char.ToLower(GetNextChar()))
-							{
-								case 'n':
-									str += '\n';
-									break;
-								case 't':
-									str += '\t';
-									break;
-								case '\\':
-									str += '\\';
-									break;
-								case '"':
-									str += '"';
-									break;
-							}
-						}
-						else if (CurrentChar == '\0')
-							throw new ParserException("Unexpected end of file");
-						else
-						{
-							str += CurrentChar;
-						}
-					}
-
-					Value = new Value(str);
-					return Token.Value;
-
 				case (char)0:
 					return Token.EOF;
 			}
@@ -345,7 +258,7 @@ namespace NTERA.Interpreter.Compiler
 
 			while (true)
 			{
-				while (IsWhitespace(GetNextChar()) && Type != LexerType.String || CurrentChar == '\r')
+				while (IsWhitespace(GetNextChar()))
 				{
 				}
 
@@ -368,8 +281,7 @@ namespace NTERA.Interpreter.Compiler
 				StringBuilder bodyBuilder = new StringBuilder(CurrentChar.ToString());
 
 				while (DetermineToken(true, false) == Token.Unknown
-					   && (!IsWhitespace(GetNextChar(true)) || Type == LexerType.String)
-					   && GetNextChar(true) != '\r')
+					   && !IsWhitespace(GetNextChar(true)))
 				{
 					bodyBuilder.Append(GetNextChar());
 				}
@@ -409,16 +321,6 @@ namespace NTERA.Interpreter.Compiler
 					continue;
 				}
 
-				if (Type == LexerType.String)
-				{
-					Value = char.IsWhiteSpace(Identifier[0])
-						? Identifier.Substring(1)
-						: Identifier;
-
-					yield return Token.Value;
-					continue;
-				}
-
 				yield return Token.Identifer;
 
 				if (CurrentChar == '\n')
@@ -445,7 +347,7 @@ namespace NTERA.Interpreter.Compiler
 			{ Token.Asterisk, 3 }, { Token.Slash, 3 },
 			{ Token.Caret, 4 }
 		};
-		
+
 		public Value Expression()
 		{
 			Stack<Value> stack = new Stack<Value>();
@@ -468,10 +370,7 @@ namespace NTERA.Interpreter.Compiler
 				}
 				else if (currentEnumerator.Current == Token.Identifer)
 				{
-					if (Type == LexerType.String)
-						stack.Push(Identifier);
-					else
-						throw new ParserException("Undeclared variable " + Identifier, TokenMarker);
+					throw new ParserException("Undeclared variable " + Identifier, TokenMarker);
 				}
 				else if (currentEnumerator.Current == Token.LParen)
 				{
@@ -481,14 +380,12 @@ namespace NTERA.Interpreter.Compiler
 					if (currentEnumerator.Current != Token.RParen)
 						throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);
 				}
-				else if (Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic()
-													  && currentEnumerator.Current.IsUnary() && (i == 0)) // || previousToken == Token.LParen))
+				else if (currentEnumerator.Current.IsArithmetic() && currentEnumerator.Current.IsUnary() && i == 0)
 				{
 					stack.Push(0);
 					operators.Push(currentEnumerator.Current);
 				}
-				else if (Type == LexerType.String && currentEnumerator.Current.IsStringOp()
-						 || Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic())
+				else if (currentEnumerator.Current.IsArithmetic())
 				{
 					while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()])
 						Operation(operators.Pop());
@@ -497,12 +394,7 @@ namespace NTERA.Interpreter.Compiler
 				else
 				{
 					if (i == 0)
-					{
-						if (Type == LexerType.String)
-							stack.Push("");
-						else
-							throw new ParserException("Empty expression", TokenMarker);
-					}
+						throw new ParserException("Empty expression", TokenMarker);
 
 					break;
 				}
@@ -514,9 +406,7 @@ namespace NTERA.Interpreter.Compiler
 			while (operators.Count > 0)
 				Operation(operators.Pop());
 
-			return Type == LexerType.String
-				? stack.Aggregate((a, b) => b.String + a.String)
-				: stack.Pop();
+			return stack.Pop();
 		}
 	}
 }

NTERA.Interpreter/Compiler/Parser.cs

@@ -697,7 +697,6 @@ namespace NTERA.Interpreter.Compiler
 				   && token != Token.EOF
 				   && token != Token.Comma
 				   && token != Token.Colon
-				   && token != Token.Format
 				   && token != Token.CloseBracket
 				   && token != Token.RParen
 				   && (useModulo || token != Token.Modulo))

NTERA.Interpreter/Compiler/Preprocessor.cs

@@ -201,13 +201,7 @@ namespace NTERA.Interpreter.Compiler
 								{
 									enumerator.MoveNext();
 
-									if (isString)
-										lexer.Type = LexerType.String;
-
-
 									defaultValue = lexer.Expression();
-
-									lexer.Type = LexerType.Both;
 								}
 								else if (enumerator.Current != Token.NewLine
 										 && enumerator.Current != Token.EOF)

NTERA.Interpreter/Compiler/Token.cs

@@ -7,10 +7,10 @@
 		Identifer,
 		Value,
 
-		[LexerCharacter('#', LexerType.Real)]
+		[LexerCharacter('#')]
 		Sharp,
 
-		[LexerCharacter('@', LexerType.Real)]
+		[LexerCharacter('@')]
 		AtSymbol,
 
 		[LexerKeyword("DIM")]
@@ -32,24 +32,25 @@
 		[LexerKeyword("FUNCTIONS")]
 		ReturnFunction,
 
-		[LexerCharacter('\n', LexerType.Both)]
+		[LexerCharacter('\n')]
 		NewLine,
 
-		[LexerCharacter('"', LexerType.Both)]
+		[LexerCharacter('"')]
 		QuotationMark,
 
-		[LexerCharacter(':', LexerType.Real)]
+		[LexerCharacter(':')]
 		Colon,
 
-		[LexerCharacter(',', LexerType.Real)]
+		[LexerCharacter(',')]
 		Comma,
-		
-		[LexerCharacter('{', LexerType.Both)]
+
+		[LexerCharacter('{')]
 		OpenBracket,
 
-		[LexerCharacter('}', LexerType.Both)]
+		[LexerCharacter('}')]
 		CloseBracket,
-		Format,
+
+		[LexerCharacter('%')]
 		Modulo,
 
 		Plus,
@@ -57,13 +58,13 @@
 		Minus,
 		Decrement,
 
-		[LexerCharacter('/', LexerType.Real)]
+		[LexerCharacter('/')]
 		Slash,
 
-		[LexerCharacter('*', LexerType.Real)]
+		[LexerCharacter('*')]
 		Asterisk,
 
-		[LexerCharacter('^', LexerType.Real)]
+		[LexerCharacter('^')]
 		Caret,
 		Equal,
 		Less,
@@ -74,13 +75,13 @@
 		Or,
 		And,
 
-		[LexerCharacter('!', LexerType.Real)]
+		[LexerCharacter('!')]
 		Not,
 
-		[LexerCharacter('(', LexerType.Real)]
+		[LexerCharacter('(')]
 		LParen,
 
-		[LexerCharacter(')', LexerType.Real)]
+		[LexerCharacter(')')]
 		RParen,
 
 		EOF = -1 //End Of File
@@ -113,10 +114,5 @@
 				   || token == Token.And
 				   || token == Token.Not;
 		}
-
-		public static bool IsStringOp(this Token token)
-		{
-			return token == Token.Plus;
-		}
 	}
 }