Implement proper CSV parsing

NTERA.Interpreter/Compiler/CSVDefinition.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+
+namespace NTERA.Interpreter.Compiler
+{
+	public class CSVDefinition
+	{
+		public Dictionary<string, string> GameBaseInfo { get; } = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
+
+		public Dictionary<string, Dictionary<string, int>> VariableIndexDictionary { get; } = new Dictionary<string, Dictionary<string, int>>(StringComparer.OrdinalIgnoreCase);
+
+		public Dictionary<string, Dictionary<int, string>> VariableDefaultValueDictionary { get; } = new Dictionary<string, Dictionary<int, string>>(StringComparer.OrdinalIgnoreCase);
+	}
+}

NTERA.Interpreter/Compiler/FunctionDefinition.cs

@@ -42,13 +42,13 @@ namespace NTERA.Interpreter.Compiler
 	{
 		public string Name { get; }
 
-		public System.ValueType ValueType { get; }
+		public ValueType ValueType { get; }
 
 		public VariableType VariableType { get; }
 
 		public Value? DefaultValue { get; }
 
-		public FunctionVariable(string name, System.ValueType valueType, VariableType variableType = VariableType.None, Value? defaultValue = null)
+		public FunctionVariable(string name, ValueType valueType, VariableType variableType = VariableType.None, Value? defaultValue = null)
 		{
 			Name = name;
 			ValueType = valueType;
@@ -63,13 +63,16 @@ namespace NTERA.Interpreter.Compiler
 
 		public string[] Indices { get; }
 
+		public bool IsArrayParameter { get; }
+
 		public Value? DefaultValue { get; }
 
-		public FunctionParameter(string name, string[] indices, Value? defaultValue = null)
+		public FunctionParameter(string name, string[] indices, Value? defaultValue = null, bool isArrayParameter = false)
 		{
 			Name = name;
 			Indices = indices;
 			DefaultValue = defaultValue;
+			IsArrayParameter = isArrayParameter;
 		}
 	}
 }

NTERA.Interpreter/Compiler/Lexer.cs

@@ -132,8 +132,10 @@ namespace NTERA.Interpreter.Compiler
 			return c == '\n' || c == '\r' || c == '\0';
 		}
 
-		private Token DetermineToken(char c)
+		private Token DetermineToken(bool peek, bool useCurrent)
 		{
+			char c = useCurrent ? currentChar : GetNextChar(peek);
+
 			if (TokenCharDictionary.TryGetValue(c, out Token charToken))
 				return charToken;
 
@@ -213,7 +215,10 @@ namespace NTERA.Interpreter.Compiler
 					if (Type == LexerType.String)
 						return Token.Unknown;
 
-					if ((source[sourceMarker.Pointer] == '+' && source[sourceMarker.Pointer + 1] == '+') || source[sourceMarker.Pointer + 2] == '+')
+					if (peek)
+						GetNextChar();
+
+					if (GetNextChar(true) == '+')
 					{
 						GetNextChar();
 						return Token.Increment;
@@ -225,7 +230,10 @@ namespace NTERA.Interpreter.Compiler
 					if (Type == LexerType.String)
 						return Token.Unknown;
 
-					if ((source[sourceMarker.Pointer] == '-' && source[sourceMarker.Pointer + 1] == '-') || source[sourceMarker.Pointer + 2] == '-')
+					if (peek)
+						GetNextChar();
+
+					if (GetNextChar(true) == '-')
 					{
 						GetNextChar();
 						return Token.Decrement;
@@ -237,25 +245,51 @@ namespace NTERA.Interpreter.Compiler
 					if (Type == LexerType.String)
 						return Token.Unknown;
 
-					if ((source[sourceMarker.Pointer] == '=' && source[sourceMarker.Pointer + 1] == '=') || source[sourceMarker.Pointer + 2] == '=')
+					if (peek)
+						GetNextChar();
+
+					if (GetNextChar(true) == '=')
 						GetNextChar();
 
 					return Token.Equal;
 
 				case '&':
-					if ((source[sourceMarker.Pointer] == '&' && source[sourceMarker.Pointer + 1] == '&') || source[sourceMarker.Pointer + 2] == '&')
+					if (peek)
+						GetNextChar();
+
+					if (GetNextChar(true) == '&')
 						GetNextChar();
 
 					return Token.And;
 
 				case '|':
-					if ((source[sourceMarker.Pointer] == '|' && source[sourceMarker.Pointer + 1] == '|') || source[sourceMarker.Pointer + 2] == '|')
+					if (peek)
+						GetNextChar();
+
+					if (GetNextChar(true) == '|')
 						GetNextChar();
 
 					return Token.Or;
 
+				case '@':
+					if (Type == LexerType.String)
+						return Token.Unknown;
+
+					if (GetNextChar(true) == '"')
+					{
+						GetNextChar();
+						goto case '"';
+					}
+
+					return Token.Function;
+
 				case '"':
+
+					//if (peek)
+					//	GetNextChar();
+
 					string str = "";
+
 					while (GetNextChar() != '"')
 					{
 						if (currentChar == '\\')
@@ -277,7 +311,7 @@ namespace NTERA.Interpreter.Compiler
 							}
 						}
 						else if (currentChar == '\0')
-							throw new Exception("Unexpected end of file");
+							throw new ParserException("Unexpected end of file");
 						else
 						{
 							str += currentChar;
@@ -306,7 +340,7 @@ namespace NTERA.Interpreter.Compiler
 
 				TokenMarker = sourceMarker;
 
-				Token token = DetermineToken(currentChar);
+				Token token = DetermineToken(false, true);
 
 				if (token == Token.EOF)
 				{
@@ -322,7 +356,7 @@ namespace NTERA.Interpreter.Compiler
 
 				StringBuilder bodyBuilder = new StringBuilder(currentChar.ToString());
 
-				while (DetermineToken(GetNextChar(true)) == Token.Unknown
+				while (DetermineToken(true, false) == Token.Unknown
 					   && (!IsWhitespace(GetNextChar(true)) || Type == LexerType.String)
 					   && GetNextChar(true) != '\r')
 				{

NTERA.Interpreter/Compiler/Parser.cs

@@ -10,15 +10,16 @@ namespace NTERA.Interpreter.Compiler
 
 		protected FunctionDefinition SelfDefinition { get; }
 
-		protected IList<FunctionDefinition> FunctionDefinitions { get; }
+		protected ICollection<FunctionDefinition> FunctionDefinitions { get; }
+		protected ICollection<FunctionDefinition> ProcedureDefinitions { get; }
 
 		protected VariableDictionary GlobalVariables { get; }
 
 		protected VariableDictionary LocalVariables { get; }
 
-		protected IList<string> StringStatements { get; }
+		protected ICollection<string> StringStatements { get; }
 
-		protected IDictionary<string, IList<IList<string>>> CsvDefinitions { get; }
+		protected CSVDefinition CsvDefinition { get; }
 
 		protected IEnumerator<Token> Enumerator { get; }
 
@@ -43,17 +44,18 @@ namespace NTERA.Interpreter.Compiler
 			Lexer.TokenMarker.Line + SelfDefinition.Position.Line - 1,
 			Lexer.TokenMarker.Column);
 
-		public Parser(string input, FunctionDefinition selfDefinition, IList<FunctionDefinition> functionDefinitions, VariableDictionary globalVariables, VariableDictionary localVariables, IList<string> stringStatements, IDictionary<string, IList<IList<string>>> csvDefinitions)
+		public Parser(string input, FunctionDefinition selfDefinition, ICollection<FunctionDefinition> functionDefinitions, ICollection<FunctionDefinition> procedureDefinitions, VariableDictionary globalVariables, VariableDictionary localVariables, ICollection<string> stringStatements, CSVDefinition csvDefinition)
 		{
 			Lexer = new Lexer(input);
 			Enumerator = Lexer.GetEnumerator();
 
 			SelfDefinition = selfDefinition;
 			FunctionDefinitions = functionDefinitions;
+			ProcedureDefinitions = procedureDefinitions;
 			GlobalVariables = globalVariables;
 			LocalVariables = localVariables;
 			StringStatements = stringStatements;
-			CsvDefinitions = csvDefinitions;
+			CsvDefinition = csvDefinition;
 		}
 
 		public IEnumerable<ExecutionNode> Parse(out List<ParserError> errors)
@@ -191,34 +193,178 @@ namespace NTERA.Interpreter.Compiler
 							Symbol = CurrentPosition
 						};
 
-						var value = Expression(out error);
-						if (error != null)
+						List<ExecutionNode> subNodes = new List<ExecutionNode>();
+
+						do
+						{
+							var value = Expression(out error);
+							if (error != null)
+								return null;
+							
+							if (Enumerator.Current == Token.Identifer)
+							{
+								if (Lexer.Identifer == "TO")
+								{
+									var value2 = Expression(out error);
+									if (error != null)
+										return null;
+
+									subNodes.Add(new ExecutionNode
+									{
+										Type = "case-to",
+										SubNodes = new[] { value, value2 }
+									});
+
+									continue;
+								}
+							}
+
+							subNodes.Add(new ExecutionNode
+							{
+								Type = "case-exact",
+								SubNodes = new[] { value }
+							});
+						} while (Enumerator.Current == Token.Comma);
+
+						if (Enumerator.Current != Token.NewLine
+							&& Enumerator.Current != Token.EOF)
+						{
+							error = new ParserError($"Unexpected token: {Enumerator.Current}", CurrentPosition);
+							return null;
+						}
+
+						node.SubNodes = subNodes.ToArray();
+						return node;
+					}
+					else if (Lexer.Identifer == "CALL"
+							 || Lexer.Identifer == "BEGIN")
+					{
+						Enumerator.MoveNext();
+
+						if (Enumerator.Current != Token.Identifer)
+						{
+							error = new ParserError($"Expecting a call to a function, got token instead: {Enumerator.Current}", CurrentPosition);
 							return null;
+						}
 
-						if (Enumerator.Current == Token.NewLine
-							|| Enumerator.Current == Token.EOF)
+						Marker symbolMarker = CurrentPosition;
+						string target = Lexer.Identifer;
+						List<ExecutionNode> parameters = new List<ExecutionNode>();
+
+						if (ProcedureDefinitions.All(x => !x.Name.Equals(target, StringComparison.OrdinalIgnoreCase)))
 						{
-							node.Metadata["casetype"] = "value";
-							node.SubNodes = new[] { value };
-							return node;
+							error = new ParserError($"Could not find procedure: {Lexer.Identifer}", CurrentPosition);
+							return null;
 						}
 
-						if (Enumerator.Current == Token.Identifer)
+						Enumerator.MoveNext();
+
+						while (Enumerator.Current != Token.NewLine
+							   && Enumerator.Current != Token.EOF)
 						{
-							if (Lexer.Identifer == "TO")
+							parameters.Add(Expression(out error));
+							if (error != null)
+							{
+								error = new ParserError($"{error.ErrorMessage} (target [{target}])", error.SymbolMarker);
+								return null;
+							}
+
+							if (Enumerator.Current != Token.Comma
+								&& Enumerator.Current != Token.NewLine
+								&& Enumerator.Current != Token.EOF)
 							{
-								var value2 = Expression(out error);
+								error = new ParserError($"Unexpected token: {Enumerator.Current}", CurrentPosition);
+								return null;
+							}
+						}
+
+						return CallMethod(target, symbolMarker, parameters.ToArray());
+					}
+					else if (Lexer.Identifer == "CALLFORM"
+							 || Lexer.Identifer == "TRYCALLFORM")
+					{
+						string statementName = Lexer.Identifer;
+
+						var node = new ExecutionNode
+						{
+							Type = "callform",
+							Metadata =
+							{
+								["try"] = statementName.StartsWith("TRY").ToString()
+							},
+							Symbol = CurrentPosition
+						};
+
+						ExecutionNode nameValue = null;
+						List<ExecutionNode> parameters = new List<ExecutionNode>();
+
+						Enumerator.MoveNext();
+
+						do
+						{
+							ExecutionNode newValue = null;
+
+							if (Enumerator.Current == Token.Identifer)
+							{
+								newValue = CreateConstant(Lexer.Identifer);
+							}
+							else if (Enumerator.Current == Token.OpenBracket)
+							{
+								newValue = Expression(out error);
 								if (error != null)
 									return null;
+							}
+							else
+							{
+								error = new ParserError($"Unexpected token: {Enumerator.Current}", CurrentPosition);
+								return null;
+							}
+
+							nameValue = nameValue == null
+								? newValue
+								: OperateNodes(nameValue, newValue, Token.Plus);
+
+							Enumerator.MoveNext();
 
-								node.Metadata["casetype"] = "to";
-								node.SubNodes = new[] { value, value2 };
-								return node;
+						} while (Enumerator.Current != Token.Comma
+								 && Enumerator.Current != Token.NewLine
+								 && Enumerator.Current != Token.EOF);
+
+
+						while (Enumerator.Current != Token.NewLine
+							   && Enumerator.Current != Token.EOF)
+						{
+							parameters.Add(Expression(out error));
+							if (error != null)
+							{
+								error = new ParserError($"{error.ErrorMessage} (statement [{statementName}])", error.SymbolMarker);
+								return null;
+							}
+
+							if (Enumerator.Current != Token.Comma
+								&& Enumerator.Current != Token.NewLine
+								&& Enumerator.Current != Token.EOF)
+							{
+								error = new ParserError($"Unexpected token: {Enumerator.Current}", CurrentPosition);
+								return null;
 							}
 						}
 
-						error = new ParserError($"Unexpected token: {Enumerator.Current}", CurrentPosition);
-						return null;
+						node.SubNodes = new[]
+						{
+							new ExecutionNode
+							{
+								Type = "name",
+								SubNodes = new[] { nameValue }
+							},
+							new ExecutionNode
+							{
+								Type = "parameters",
+								SubNodes = parameters.ToArray()
+							},
+						};
+
+						return node;
 					}
 					else //treat as statement
 					{
@@ -347,16 +493,10 @@ namespace NTERA.Interpreter.Compiler
 				}
 				else if (token == Token.Identifer)
 				{
-					IList<IList<string>> csvTable = CsvDefinitions
-													.Where(x => x.Key.IndexOf(variableName, StringComparison.OrdinalIgnoreCase) >= 0)
-													.OrderBy(x => x.Key.Equals(variableName, StringComparison.OrdinalIgnoreCase) ? 1 : 2)
-													.FirstOrDefault().Value;
-
-					IList<string> alias = csvTable?.FirstOrDefault(x => x.Count > 1 && x[1] == Lexer.Identifer);
-
-					if (alias != null)
+					if (CsvDefinition.VariableIndexDictionary.TryGetValue(variableName, out var varTable)
+						&& varTable.TryGetValue(Lexer.Identifer, out int index))
 					{
-						indices.Add(CreateConstant(int.Parse(alias[0])));
+						indices.Add(CreateConstant(index));
 						continue;
 					}
 
@@ -379,7 +519,7 @@ namespace NTERA.Interpreter.Compiler
 
 					if (FunctionDefinitions.Any(x => x.Name == Lexer.Identifer))
 					{
-						indices.Add(Expression(out error));
+						indices.Add(GetFunction(out error));
 						if (error != null)
 							return null;
 
@@ -445,8 +585,14 @@ namespace NTERA.Interpreter.Compiler
 				return null;
 			}
 
+			if (hasPeeked)
+			{
+				GetNextToken();
+			}
+
 			var functionDefinition = FunctionDefinitions.FirstOrDefault(x => x.Name == functionName
-																			 && x.Parameters.Length >= parameters.Count);
+																			 && (x.Parameters.Length >= parameters.Count
+																				 || x.Parameters.Any(y => y.IsArrayParameter)));
 
 			if (functionDefinition == null)
 			{
@@ -568,6 +714,8 @@ namespace NTERA.Interpreter.Compiler
 					}
 					else
 					{
+						break;
+						//this should be converted into a warning
 						error = new ParserError($"Unknown identifier: {Lexer.Identifer}", CurrentPosition);
 						return null;
 					}
@@ -622,6 +770,12 @@ namespace NTERA.Interpreter.Compiler
 					return null;
 			}
 
+			if (!operands.Any())
+			{
+				error = new ParserError("Invalid expression - Empty operand stack", CurrentPosition);
+				return null;
+			}
+
 			return operands.Pop();
 		}
 

NTERA.Interpreter/Compiler/Preprocessor.cs

@@ -1,4 +1,5 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
 using System.Linq;
 
 namespace NTERA.Interpreter.Compiler
@@ -228,6 +229,16 @@ namespace NTERA.Interpreter.Compiler
 							}
 						}
 					}
+					else
+					{
+						//resynchronize to next line
+						while (enumerator.Current != Token.NewLine
+							   && enumerator.Current != Token.EOF
+							   && enumerator.MoveNext())
+						{
+							
+						}
+					}
 				} while (enumerator.MoveNext());
 			}
 
@@ -236,7 +247,7 @@ namespace NTERA.Interpreter.Compiler
 			return procs;
 		}
 
-		public static IList<IList<string>> ProcessCSV(IEnumerable<string> lines)
+		private static IList<IList<string>> SplitCSV(IEnumerable<string> lines)
 		{
 			List<IList<string>> csv = new List<IList<string>>();
 
@@ -260,5 +271,100 @@ namespace NTERA.Interpreter.Compiler
 
 			return csv;
 		}
+
+		private static Dictionary<string, string[]> NameIndexDictionary = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
+		{
+			["ITEM"] = new[] { "ITEM", "ITEMSALES", "ITEMPRICE" },
+			["BASE"] = new[] { "BASE", "LOSEBASE", "MAXBASE", "DOWNBASE" },
+			["ABL"] = new[] { "ABL" },
+			["TALENT"] = new[] { "TALENT" },
+			["EXP"] = new[] { "EXP" },
+			["MARK"] = new[] { "MARK" },
+			["PALAM"] = new[] { "PALAM", "UP", "DOWN", "JUEL", "GOTJUEL", "CUP", "CDOWN" },
+			["STAIN"] = new[] { "STAIN" },
+			["SOURCE"] = new[] { "SOURCE" },
+			["EX"] = new[] { "EX", "NOWEX" },
+			["TEQUIP"] = new[] { "TEQUIP" },
+			["EQUIP"] = new[] { "EQUIP" },
+			["FLAG"] = new[] { "FLAG" },
+			["TFLAG"] = new[] { "TFLAG" },
+			["CFLAG"] = new[] { "CFLAG" },
+			["STRNAME"] = new[] { "STR" },
+			["SAVESTR"] = new[] { "SAVESTR" },
+			["TCVAR"] = new[] { "TCVAR" },
+			["TSTR"] = new[] { "TSTR" },
+			["CSTR"] = new[] { "CSTR" },
+			["CDFLAG1"] = new[] { "CDFLAG" },
+			["CDFLAG2"] = new[] { "CDFLAG" },
+			["GLOBAL"] = new[] { "GLOBAL" },
+			["GLOBALS"] = new[] { "GLOBALS" },
+		};
+
+		public static void ProcessCSV(CSVDefinition targetDefinition, string filename, IEnumerable<string> lines)
+		{
+			if (filename.EndsWith("_TR", StringComparison.OrdinalIgnoreCase))
+				return;
+
+			if (filename.Equals("VariableSize", StringComparison.OrdinalIgnoreCase))
+				return;
+
+			if (filename.Equals("_Replace", StringComparison.OrdinalIgnoreCase))
+				return;
+
+			var csv = SplitCSV(lines);
+
+			void AddVariableIndices(string variableName)
+			{
+				Dictionary<string, int> varIndices = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);
+
+				foreach (var line in csv)
+					if (!string.IsNullOrWhiteSpace(line[1]))
+						varIndices[line[1]] = int.Parse(line[0]);
+
+				targetDefinition.VariableIndexDictionary[variableName] = varIndices;
+			}
+
+			if (filename.Equals("GameBase", StringComparison.OrdinalIgnoreCase))
+			{
+				foreach (var line in csv)
+					targetDefinition.GameBaseInfo.Add(line[0], line[1]);
+
+				return;
+			}
+
+			if (NameIndexDictionary.TryGetValue(filename, out var variables))
+			{
+				foreach (var variable in variables)
+					AddVariableIndices(variable);
+
+				return;
+			}
+
+			if (filename.Equals("STR", StringComparison.OrdinalIgnoreCase))
+			{
+				Dictionary<int, string> strDefaultValues = new Dictionary<int, string>();
+
+				foreach (var line in csv)
+					strDefaultValues.Add(int.Parse(line[0]), line[1]);
+
+				targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues;
+
+				return;
+			}
+
+			if (filename.StartsWith("CHARA", StringComparison.OrdinalIgnoreCase))
+			{
+				//Dictionary<int, string> strDefaultValues = new Dictionary<int, string>();
+
+				//foreach (var line in csv)
+				//	strDefaultValues.Add(int.Parse(line[0]), line[1]);
+
+				//targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues;
+
+				return;
+			}
+
+			//AddVariableIndices(Path.GetFileNameWithoutExtension(filename));
+		}
 	}
 }

NTERA.Interpreter/Compiler/Token.cs

@@ -10,7 +10,7 @@
 		[LexerCharacter('#', LexerType.Real)]
 		Sharp,
 
-		[LexerCharacter('@', LexerType.Real)]
+		//[LexerCharacter('@', LexerType.Real)]
 		Function,
 
 		[LexerKeyword("DIM")]

NTERA.Interpreter/NTERA.Interpreter.csproj

@@ -38,6 +38,7 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Compiler\CSVDefinition.cs" />
     <Compile Include="Compiler\ExecutionNode.cs" />
     <Compile Include="Compiler\FunctionDefinition.cs" />
     <Compile Include="Compiler\Parser.cs" />