using System; using System.Collections.Generic; using System.Linq; using System.Text; namespace NTERA.Engine.Compiler { public static class Preprocessor { public static IEnumerable PreprocessHeaderFile(string contents) { Lexer lexer = new Lexer(contents); List constantDefinitions = new List(); using (var enumerator = lexer.GetEnumerator()) { do { if (lexer.TokenMarker.Column != 1) continue; if (enumerator.Current == Token.Sharp) { enumerator.MoveNext(); switch (enumerator.Current) { case Token.Dims: case Token.Dim: { bool isString = enumerator.Current != Token.Dim; enumerator.MoveNext(); VariableType variableType = VariableType.None; while (enumerator.Current == Token.Const || enumerator.Current == Token.Ref || enumerator.Current == Token.Dynamic || (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("GLOBAL", StringComparison.OrdinalIgnoreCase)) || (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("CHARADATA", StringComparison.OrdinalIgnoreCase)) || (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("SAVEDATA", StringComparison.OrdinalIgnoreCase))) { if (enumerator.Current == Token.Const) variableType |= VariableType.Constant; else if (enumerator.Current == Token.Ref) variableType |= VariableType.Reference; else if (enumerator.Current == Token.Dynamic) variableType |= VariableType.Dynamic; else if (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("GLOBAL", StringComparison.OrdinalIgnoreCase)) variableType |= VariableType.SaveData; else if (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("CHARADATA", StringComparison.OrdinalIgnoreCase)) variableType |= VariableType.CharaData; else if (enumerator.Current == Token.Identifer && lexer.Identifier.Equals("SAVEDATA", StringComparison.OrdinalIgnoreCase)) variableType |= VariableType.Global; enumerator.MoveNext(); } string variable = lexer.Identifier; enumerator.MoveNext(); Value? defaultValue = null; if (enumerator.Current == Token.Comma) { while (enumerator.MoveNext() && enumerator.Current != Token.Equal && enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) { //arraySize = (int)lexer.Expression().Real; //the array size goes here, but we ignore it since it's useless to us } } if (enumerator.Current == Token.Equal) { enumerator.MoveNext(); defaultValue = ConstantExpression(lexer, constantDefinitions); } else if (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) { throw new ParserException("Invalid function declaration", lexer.TokenMarker); } var functionDefinition = new FunctionVariable(variable, isString ? ValueType.String : ValueType.Real, variableType, defaultValue); constantDefinitions.Add(functionDefinition); yield return functionDefinition; break; } } } else { //resynchronize to next line while (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF && enumerator.MoveNext()) { } } } while (enumerator.MoveNext()); } } public static IDictionary PreprocessCodeFile(string contents, string filename, ICollection constantDefinitions) { Dictionary procedures = new Dictionary(); Lexer lexer = new Lexer(contents); Marker startMarker = lexer.TokenMarker; string currentDefinitionName = null; List currentDefinitionParameters = new List(); List currentDefinitionVariables = new List(); bool isReturnFunction = false; void Commit() { if (currentDefinitionName != null) { string procBody = contents.Substring(startMarker.Pointer, lexer.TokenMarker.Pointer - startMarker.Pointer); var definition = new FunctionDefinition(currentDefinitionName, currentDefinitionParameters.ToArray(), currentDefinitionVariables.ToArray(), isReturnFunction, filename, startMarker); procedures.Add(definition, procBody); isReturnFunction = false; currentDefinitionName = null; currentDefinitionParameters.Clear(); } } using (var enumerator = lexer.GetEnumerator()) { do { if (lexer.TokenMarker.Column != 1) continue; if (enumerator.Current == Token.AtSymbol) { Commit(); startMarker = lexer.TokenMarker; enumerator.MoveNext(); if (enumerator.Current != Token.Identifer) throw new ParserException("Invalid function declaration - Expected an identifier", lexer.TokenMarker); currentDefinitionName = lexer.Identifier; enumerator.MoveNext(); if (enumerator.Current == Token.NewLine || enumerator.Current == Token.EOF) continue; if (enumerator.Current != Token.LParen && enumerator.Current != Token.Comma) throw new ParserException("Invalid function declaration", lexer.TokenMarker); enumerator.MoveNext(); if (enumerator.Current != Token.Identifer && enumerator.Current != Token.RParen) throw new ParserException("Invalid function declaration", lexer.TokenMarker); while (enumerator.Current == Token.Identifer) { string parameterName = lexer.Identifier; List indices = new List(); Value? defaultValue = null; enumerator.MoveNext(); while (enumerator.Current == Token.Colon && enumerator.MoveNext()) { if (enumerator.Current == Token.Value) { indices.Add(lexer.Value.Type == ValueType.Real ? ((int)lexer.Value).ToString() : lexer.Value.String); } else if (enumerator.Current == Token.Identifer) { indices.Add(lexer.Identifier); } enumerator.MoveNext(); } if (enumerator.Current == Token.Equal) { enumerator.MoveNext(); defaultValue = ConstantExpression(lexer, constantDefinitions); } if (enumerator.Current == Token.Comma || enumerator.Current == Token.RParen) { enumerator.MoveNext(); } else if (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) throw new ParserException("Invalid function declaration", lexer.TokenMarker); currentDefinitionParameters.Add(new FunctionParameter(parameterName, indices.ToArray(), defaultValue)); } if (enumerator.Current == Token.RParen) enumerator.MoveNext(); if (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) throw new ParserException("Invalid function declaration", lexer.TokenMarker); } else if (enumerator.Current == Token.Sharp) { enumerator.MoveNext(); switch (enumerator.Current) { case Token.Dims: case Token.Dim: { bool isString = enumerator.Current != Token.Dim; enumerator.MoveNext(); VariableType variableType = VariableType.None; while (enumerator.Current == Token.Const || enumerator.Current == Token.Ref || enumerator.Current == Token.Dynamic) { if (enumerator.Current == Token.Const) variableType |= VariableType.Constant; else if (enumerator.Current == Token.Ref) variableType |= VariableType.Reference; else if (enumerator.Current == Token.Dynamic) variableType |= VariableType.Dynamic; enumerator.MoveNext(); } string variable = lexer.Identifier; enumerator.MoveNext(); Value? defaultValue = null; if (enumerator.Current == Token.Comma) { while (enumerator.MoveNext() && enumerator.Current != Token.Equal && enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) { //arraySize = (int)lexer.Expression().Real; //the array size goes here, but we ignore it since it's useless to us } } if (enumerator.Current == Token.Equal) { enumerator.MoveNext(); defaultValue = ConstantExpression(lexer, constantDefinitions); } else if (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF) { throw new ParserException("Invalid function declaration", lexer.TokenMarker); } currentDefinitionVariables.Add(new FunctionVariable(variable, isString ? ValueType.String : ValueType.Real, variableType, defaultValue)); break; } case Token.ReturnFunction: { isReturnFunction = true; break; } } } else { //resynchronize to next line while (enumerator.Current != Token.NewLine && enumerator.Current != Token.EOF && enumerator.MoveNext()) { } } } while (enumerator.MoveNext()); } Commit(); return procedures; } private static IList> SplitCSV(IEnumerable lines) { List> csv = new List>(); foreach (var line in lines) { if (string.IsNullOrWhiteSpace(line) || line[0] == ';') continue; string newLine = line; int commentIndex = line.IndexOf(';'); if (commentIndex >= 0) newLine = line.Substring(0, commentIndex); string[] split = newLine.Split(new[] { ',' }, StringSplitOptions.None); if (split.Length == 1) continue; csv.Add(split.ToList()); } return csv; } private static Dictionary NameIndexDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase) { ["ITEM"] = new[] { "ITEM", "ITEMSALES", "ITEMPRICE" }, ["BASE"] = new[] { "BASE", "LOSEBASE", "MAXBASE", "DOWNBASE" }, ["ABL"] = new[] { "ABL" }, ["TALENT"] = new[] { "TALENT" }, ["EXP"] = new[] { "EXP" }, ["MARK"] = new[] { "MARK" }, ["PALAM"] = new[] { "PALAM", "UP", "DOWN", "JUEL", "GOTJUEL", "CUP", "CDOWN" }, ["STAIN"] = new[] { "STAIN" }, ["SOURCE"] = new[] { "SOURCE" }, ["EX"] = new[] { "EX", "NOWEX" }, ["TEQUIP"] = new[] { "TEQUIP" }, ["EQUIP"] = new[] { "EQUIP" }, ["FLAG"] = new[] { "FLAG" }, ["TFLAG"] = new[] { "TFLAG" }, ["CFLAG"] = new[] { "CFLAG" }, ["STRNAME"] = new[] { "STR" }, ["SAVESTR"] = new[] { "SAVESTR" }, ["TCVAR"] = new[] { "TCVAR" }, ["TSTR"] = new[] { "TSTR" }, ["CSTR"] = new[] { "CSTR" }, ["CDFLAG1"] = new[] { "CDFLAG" }, ["CDFLAG2"] = new[] { "CDFLAG" }, ["GLOBAL"] = new[] { "GLOBAL" }, ["GLOBALS"] = new[] { "GLOBALS" }, }; public static void ProcessCSV(CSVDefinition targetDefinition, string filename, IEnumerable lines) { if (filename.EndsWith("_TR", StringComparison.OrdinalIgnoreCase)) return; if (filename.Equals("VariableSize", StringComparison.OrdinalIgnoreCase)) return; if (filename.Equals("_Replace", StringComparison.OrdinalIgnoreCase)) return; var csv = SplitCSV(lines); void AddVariableIndices(string variableName) { Dictionary varIndices = new Dictionary(StringComparer.OrdinalIgnoreCase); foreach (var line in csv) if (!string.IsNullOrWhiteSpace(line[1])) varIndices[line[1]] = int.Parse(line[0]); targetDefinition.VariableIndexDictionary[variableName] = varIndices; } if (filename.Equals("GameBase", StringComparison.OrdinalIgnoreCase)) { foreach (var line in csv) targetDefinition.GameBaseInfo.Add(line[0], line[1]); return; } if (NameIndexDictionary.TryGetValue(filename, out var variables)) { foreach (var variable in variables) AddVariableIndices(variable); return; } if (filename.Equals("STR", StringComparison.OrdinalIgnoreCase)) { Dictionary strDefaultValues = new Dictionary(); foreach (var line in csv) strDefaultValues.Add(int.Parse(line[0]), line[1]); targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues; return; } if (filename.StartsWith("CHARA", StringComparison.OrdinalIgnoreCase)) { //Dictionary strDefaultValues = new Dictionary(); //foreach (var line in csv) // strDefaultValues.Add(int.Parse(line[0]), line[1]); //targetDefinition.VariableDefaultValueDictionary["STR"] = strDefaultValues; return; } //AddVariableIndices(Path.GetFileNameWithoutExtension(filename)); } private static readonly Dictionary OrderOfOps = new Dictionary { { Token.Or, 0 }, { Token.And, 0 }, { Token.Equal, 1 }, { Token.NotEqual, 1 }, { Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 }, { Token.MoreEqual, 1 }, { Token.Plus, 2 }, { Token.Minus, 2 }, { Token.Asterisk, 3 }, { Token.Slash, 3 }, { Token.Caret, 4 } }; public static Value ConstantExpression(Lexer lexer, ICollection constantDefinitions = null) { IEnumerator currentEnumerator = lexer.GetEnumerator(); Stack stack = new Stack(); Stack operators = new Stack(); void Operation(Token token) { if (stack.Count < 2) throw new ParserException("Not enough operands to perform operation", lexer.TokenMarker); Value b = stack.Pop(); Value a = stack.Pop(); Value result = a.Operate(b, token); stack.Push(result); } int i = 0; while (true) { if (currentEnumerator.Current == Token.Value) { stack.Push(lexer.Value); } else if (currentEnumerator.Current == Token.QuotationMark) { StringBuilder builder = new StringBuilder(); char stringChar; while ((stringChar = lexer.GetNextChar()) != '"') builder.Append(stringChar); stack.Push(builder.ToString()); } else if (currentEnumerator.Current == Token.Identifer) { var variable = constantDefinitions?.FirstOrDefault(x => x.Name.Equals(lexer.Identifier, StringComparison.OrdinalIgnoreCase) && x.VariableType.HasFlag(VariableType.Constant)); if (variable == null) throw new ParserException("Undeclared variable " + lexer.Identifier, lexer.TokenMarker); stack.Push(variable.CalculatedValue); } else if (currentEnumerator.Current == Token.LParen) { currentEnumerator.MoveNext(); stack.Push(ConstantExpression(lexer)); if (currentEnumerator.Current != Token.RParen) throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", lexer.TokenMarker); } else if (currentEnumerator.Current.IsArithmetic() && currentEnumerator.Current.IsUnary() && i == 0) { stack.Push(0); operators.Push(currentEnumerator.Current); } else if (currentEnumerator.Current.IsArithmetic()) { while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()]) Operation(operators.Pop()); operators.Push(currentEnumerator.Current); } else { if (i == 0) throw new ParserException("Empty expression", lexer.TokenMarker); break; } i++; currentEnumerator.MoveNext(); } while (operators.Count > 0) Operation(operators.Pop()); if (stack.Count == 0) throw new ParserException("Empty expression", lexer.TokenMarker); return stack.Pop(); } } }