NTERA/NTERA.Interpreter/Compiler/Preprocessor.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;

namespace NTERA.Interpreter.Compiler
{
	public static class Preprocessor
	{
		public static IEnumerable<FunctionVariable> PreprocessHeaderFile(string contents)
		{
			Lexer lexer = new Lexer(contents);
			List<FunctionVariable> constantDefinitions = new List<FunctionVariable>();

			using (var enumerator = lexer.GetEnumerator())
			{
				do
				{
					if (lexer.TokenMarker.Column != 1)
						continue;
					
					if (enumerator.Current == Token.Sharp)
					{
						enumerator.MoveNext();

						switch (enumerator.Current)
						{
							case Token.Dims:
							case Token.Dim:
								{
									bool isString = enumerator.Current != Token.Dim;

									enumerator.MoveNext();
									VariableType variableType = VariableType.None;

									while (enumerator.Current == Token.Const
										   || enumerator.Current == Token.Ref
										   || enumerator.Current == Token.Dynamic
										   || enumerator.Current == Token.SaveData
										   || enumerator.Current == Token.CharaData
										   || enumerator.Current == Token.Global)
									{
										if (enumerator.Current == Token.Const)
											variableType |= VariableType.Constant;
										else if (enumerator.Current == Token.Ref)
											variableType |= VariableType.Reference;
										else if (enumerator.Current == Token.Dynamic)
											variableType |= VariableType.Dynamic;
										else if (enumerator.Current == Token.SaveData)
											variableType |= VariableType.SaveData;
										else if (enumerator.Current == Token.CharaData)
											variableType |= VariableType.CharaData;
										else if (enumerator.Current == Token.Global)
											variableType |= VariableType.Global;

										enumerator.MoveNext();
									}

									string variable = lexer.Identifier;

									enumerator.MoveNext();
									Value? defaultValue = null;

									if (enumerator.Current == Token.Comma)
									{
										while (enumerator.MoveNext()
											   && enumerator.Current != Token.Equal
											   && enumerator.Current != Token.NewLine
											   && enumerator.Current != Token.EOF)
										{
											//arraySize = (int)lexer.Expression().Real;

											//the array size goes here, but we ignore it since it's useless to us
										}
									}

									if (enumerator.Current == Token.Equal)
									{
										enumerator.MoveNext();

										defaultValue = ConstantExpression(lexer, constantDefinitions);
									}
									else if (enumerator.Current != Token.NewLine
											 && enumerator.Current != Token.EOF)
									{
										throw new ParserException("Invalid function declaration", lexer.TokenMarker);
									}

									var functionDefinition = new FunctionVariable(variable,
										isString ? ValueType.String : ValueType.Real,
										variableType,
										defaultValue);

									constantDefinitions.Add(functionDefinition);
									yield return functionDefinition;

									break;
								}
						}
					}
					else
					{
						//resynchronize to next line
						while (enumerator.Current != Token.NewLine
							   && enumerator.Current != Token.EOF
							   && enumerator.MoveNext())
						{
						}
					}
				} while (enumerator.MoveNext());
			}
		}

		public static IDictionary<FunctionDefinition, string> PreprocessCodeFile(string contents, string filename, ICollection<FunctionVariable> constantDefinitions)
		{
			Dictionary<FunctionDefinition, string> procedures = new Dictionary<FunctionDefinition, string>();

			Lexer lexer = new Lexer(contents);

			Marker startMarker = lexer.TokenMarker;
			string currentDefinitionName = null;
			List<FunctionParameter> currentDefinitionParameters = new List<FunctionParameter>();
			List<FunctionVariable> currentDefinitionVariables = new List<FunctionVariable>();
			bool isReturnFunction = false;

			void Commit()
			{
				if (currentDefinitionName != null)
				{
					string procBody = contents.Substring(startMarker.Pointer,
						lexer.TokenMarker.Pointer - startMarker.Pointer);

					var definition = new FunctionDefinition(currentDefinitionName,
						currentDefinitionParameters.ToArray(),
						currentDefinitionVariables.ToArray(),
						isReturnFunction,
						filename,
						startMarker);

					procedures.Add(definition, procBody);

					isReturnFunction = false;
					currentDefinitionName = null;
					currentDefinitionParameters.Clear();
				}
			}

			using (var enumerator = lexer.GetEnumerator())
			{
				do
				{
					if (lexer.TokenMarker.Column != 1)
						continue;

					if (enumerator.Current == Token.AtSymbol)
					{
						Commit();

						startMarker = lexer.TokenMarker;

						enumerator.MoveNext();
						if (enumerator.Current != Token.Identifer)
							throw new ParserException("Invalid function declaration - Expected an identifier", lexer.TokenMarker);

						currentDefinitionName = lexer.Identifier;

						enumerator.MoveNext();
						if (enumerator.Current == Token.NewLine
							|| enumerator.Current == Token.EOF)
							continue;

						if (enumerator.Current != Token.LParen
							&& enumerator.Current != Token.Comma)
							throw new ParserException("Invalid function declaration", lexer.TokenMarker);

						enumerator.MoveNext();

						if (enumerator.Current != Token.Identifer
							&& enumerator.Current != Token.RParen)
							throw new ParserException("Invalid function declaration", lexer.TokenMarker);

						while (enumerator.Current == Token.Identifer)
						{
							string parameterName = lexer.Identifier;
							List<string> indices = new List<string>();
							Value? defaultValue = null;

							enumerator.MoveNext();

							while (enumerator.Current == Token.Colon
								   && enumerator.MoveNext())
							{
								if (enumerator.Current == Token.Value)
								{
									indices.Add(lexer.Value.Type == ValueType.Real
										? ((int)lexer.Value).ToString()
										: lexer.Value.String);
								}
								else if (enumerator.Current == Token.Identifer)
								{
									indices.Add(lexer.Identifier);
								}

								enumerator.MoveNext();
							}

							if (enumerator.Current == Token.Equal)
							{
								enumerator.MoveNext();

								defaultValue = ConstantExpression(lexer);

								//enumerator.MoveNext();
							}

							if (enumerator.Current == Token.Comma
								|| enumerator.Current == Token.RParen)
							{
								enumerator.MoveNext();
							}
							else if (enumerator.Current != Token.NewLine
									 && enumerator.Current != Token.EOF)
								throw new ParserException("Invalid function declaration", lexer.TokenMarker);

							currentDefinitionParameters.Add(new FunctionParameter(parameterName, indices.ToArray(), defaultValue));
						}

						if (enumerator.Current == Token.RParen)
							enumerator.MoveNext();

						if (enumerator.Current != Token.NewLine
							&& enumerator.Current != Token.EOF)
							throw new ParserException("Invalid function declaration", lexer.TokenMarker);
					}
					else if (enumerator.Current == Token.Sharp)
					{
						enumerator.MoveNext();

						switch (enumerator.Current)
						{
							case Token.Dims:
							case Token.Dim:
							{
								bool isString = enumerator.Current != Token.Dim;

								enumerator.MoveNext();
								VariableType variableType = VariableType.None;

								while (enumerator.Current == Token.Const
									   || enumerator.Current == Token.Ref
									   || enumerator.Current == Token.Dynamic)
								{
									if (enumerator.Current == Token.Const)
										variableType |= VariableType.Constant;
									else if (enumerator.Current == Token.Ref)
										variableType |= VariableType.Reference;
									else if (enumerator.Current == Token.Dynamic)
										variableType |= VariableType.Dynamic;

									enumerator.MoveNext();
								}

								string variable = lexer.Identifier;

								enumerator.MoveNext();
								Value? defaultValue = null;

								if (enumerator.Current == Token.Comma)
								{
									while (enumerator.MoveNext()
										   && enumerator.Current != Token.Equal
										   && enumerator.Current != Token.NewLine
										   && enumerator.Current != Token.EOF)
									{
										//arraySize = (int)lexer.Expression().Real;

										//the array size goes here, but we ignore it since it's useless to us
										}
									}

								if (enumerator.Current == Token.Equal)
								{
									enumerator.MoveNext();

									defaultValue = ConstantExpression(lexer, constantDefinitions);
								}
								else if (enumerator.Current != Token.NewLine
										 && enumerator.Current != Token.EOF)
								{
									throw new ParserException("Invalid function declaration", lexer.TokenMarker);
								}

								currentDefinitionVariables.Add(new FunctionVariable(variable,
									isString ? ValueType.String : ValueType.Real,
									variableType,
									defaultValue));

								break;
							}
							case Token.ReturnFunction:
							{
								isReturnFunction = true;
								break;
							}
						}
					}
					else
					{
						//resynchronize to next line
						while (enumerator.Current != Token.NewLine
							   && enumerator.Current != Token.EOF
							   && enumerator.MoveNext())
						{
						}
					}
				} while (enumerator.MoveNext());
			}

			Commit();

			return procedures;
		}

		private static IList<IList<string>> SplitCSV(IEnumerable<string> lines)
		{
			List<IList<string>> csv = new List<IList<string>>();

			foreach (var line in lines)
			{
				if (string.IsNullOrWhiteSpace(line)
					|| line[0] == ';')
					continue;

				string newLine = line;

				int commentIndex = line.IndexOf(';');

				if (commentIndex >= 0)
					newLine = line.Substring(0, commentIndex);

				string[] split = newLine.Split(new[] { ',' }, StringSplitOptions.None);

				if (split.Length == 1)
					continue;

				csv.Add(split.ToList());
			}

			return csv;
		}

		private static Dictionary<string, string[]> NameIndexDictionary = new Dictionary<string, string[]>(StringComparer.OrdinalIgnoreCase)
		{
			["ITEM"] = new[] { "ITEM", "ITEMSALES", "ITEMPRICE" },
			["BASE"] = new[] { "BASE", "LOSEBASE", "MAXBASE", "DOWNBASE" },
			["ABL"] = new[] { "ABL" },
			["TALENT"] = new[] { "TALENT" },
			["EXP"] = new[] { "EXP" },
			["MARK"] = new[] { "MARK" },
			["PALAM"] = new[] { "PALAM", "UP", "DOWN", "JUEL", "GOTJUEL", "CUP", "CDOWN" },
			["STAIN"] = new[] { "STAIN" },
			["SOURCE"] = new[] { "SOURCE" },
			["EX"] = new[] { "EX", "NOWEX" },
			["TEQUIP"] = new[] { "TEQUIP" },
			["EQUIP"] = new[] { "EQUIP" },
			["FLAG"] = new[] { "FLAG" },
			["TFLAG"] = new[] { "TFLAG" },
			["CFLAG"] = new[] { "CFLAG" },
			["STRNAME"] = new[] { "STR" },
			["SAVESTR"] = new[] { "SAVESTR" },
			["TCVAR"] = new[] { "TCVAR" },
			["TSTR"] = new[] { "TSTR" },
			["CSTR"] = new[] { "CSTR" },
			["CDFLAG1"] = new[] { "CDFLAG" },
			["CDFLAG2"] = new[] { "CDFLAG" },
			["GLOBAL"] = new[] { "GLOBAL" },
			["GLOBALS"] = new[] { "GLOBALS" },
		};

		public static void ProcessCSV(CSVDefinition targetDefinition, string filename, IEnumerable<string> lines)
		{
			if (filename.EndsWith("_TR", StringComparison.OrdinalIgnoreCase))
				return;

			if (filename.Equals("VariableSize", StringComparison.OrdinalIgnoreCase))
				return;

			if (filename.Equals("_Replace", StringComparison.OrdinalIgnoreCase))
				return;

			var csv = SplitCSV(lines);

			void AddVariableIndices(string variableName)
			{
				Dictionary<string, int> varIndices = new Dictionary<string, int>(StringComparer.OrdinalIgnoreCase);

				foreach (var line in csv)
					if (!string.IsNullOrWhiteSpace(line[1]))
						varIndices[line[1]] = int.Parse(line[0]);

				targetDefinition.VariableIndexDictionary[variableName] = varIndices;
			}

			if (filename.Equals("GameBase", StringComparison.OrdinalIgnoreCase))
			{
				foreach (var line in csv)
					targetDefinition.GameBaseInfo.Add(line[0], line[1]);

				return;
			}

			if (NameIndexDictionary.TryGetValue(filename, out var variables))
			{
				foreach (var variable in variables)
					AddVariableIndices(variable);

				return;
			}

			if (filename.Equals("STR", StringComparison.OrdinalIgnoreCase))
			{
				Dictionary<int, string> strDefaultValues = new Dictionary<int, string>();

				foreach (var line in csv)
					strDefaultValues.Add(int.Parse(line[0]), line[1]);

				targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues;

				return;
			}

			if (filename.StartsWith("CHARA", StringComparison.OrdinalIgnoreCase))
			{
				//Dictionary<int, string> strDefaultValues = new Dictionary<int, string>();

				//foreach (var line in csv)
				//	strDefaultValues.Add(int.Parse(line[0]), line[1]);

				//targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues;

				return;
			}

			//AddVariableIndices(Path.GetFileNameWithoutExtension(filename));
		}

		private static readonly Dictionary<Token, int> OrderOfOps = new Dictionary<Token, int>
		{
			{ Token.Or, 0 }, { Token.And, 0 },
			{ Token.Equal, 1 }, { Token.NotEqual, 1 },
			{ Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 }, { Token.MoreEqual, 1 },
			{ Token.Plus, 2 }, { Token.Minus, 2 },
			{ Token.Asterisk, 3 }, { Token.Slash, 3 },
			{ Token.Caret, 4 }
		};

		public static Value ConstantExpression(Lexer lexer, ICollection<FunctionVariable> constantDefinitions = null)
		{
			IEnumerator<Token> currentEnumerator = lexer.GetEnumerator();

			Stack<Value> stack = new Stack<Value>();
			Stack<Token> operators = new Stack<Token>();

			void Operation(Token token)
			{
				Value b = stack.Pop();
				Value a = stack.Pop();
				Value result = a.Operate(b, token);
				stack.Push(result);
			}

			int i = 0;
			while (true)
			{
				if (currentEnumerator.Current == Token.Value)
				{
					stack.Push(lexer.Value);
				}
				else if (currentEnumerator.Current == Token.QuotationMark)
				{
					StringBuilder builder = new StringBuilder();
					char stringChar;

					while ((stringChar = lexer.GetNextChar()) != '"')
						builder.Append(stringChar);

					stack.Push(builder.ToString());
				}
				else if (currentEnumerator.Current == Token.Identifer)
				{
					var variable = constantDefinitions?.FirstOrDefault(x => x.Name == lexer.Identifier && x.VariableType.HasFlag(VariableType.Constant));

					if (variable == null)
						throw new ParserException("Undeclared variable " + lexer.Identifier, lexer.TokenMarker);

					var value = variable.DefaultValue ?? variable.ValueType == ValueType.String ? (Value)"" : 0;

					stack.Push(value);
				}
				else if (currentEnumerator.Current == Token.LParen)
				{
					currentEnumerator.MoveNext();
					stack.Push(ConstantExpression(lexer));

					if (currentEnumerator.Current != Token.RParen)
						throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", lexer.TokenMarker);
				}
				else if (currentEnumerator.Current.IsArithmetic() && currentEnumerator.Current.IsUnary() && i == 0)
				{
					stack.Push(0);
					operators.Push(currentEnumerator.Current);
				}
				else if (currentEnumerator.Current.IsArithmetic())
				{
					while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()])
						Operation(operators.Pop());
					operators.Push(currentEnumerator.Current);
				}
				else
				{
					if (i == 0)
						throw new ParserException("Empty expression", lexer.TokenMarker);

					break;
				}

				i++;
				currentEnumerator.MoveNext();
			}

			while (operators.Count > 0)
				Operation(operators.Pop());

			return stack.Pop();
		}
	}
}