NTERA/NTERA.Compiler/Compiler.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using NTERA.Engine;
using NTERA.Engine.Compiler;

namespace NTERA.Compiler
{
	public class Compiler
	{
		public string InputDirectory { get; }

		public List<Tuple<ParserError, string>> Errors { get; } = new List<Tuple<ParserError, string>>();
		public List<Tuple<ParserError, string>> Warnings { get; } = new List<Tuple<ParserError, string>>();

		public Dictionary<FunctionDefinition, string> DeclaredFunctions = new Dictionary<FunctionDefinition, string>();

		public int Threads { get; set; }

		public Compiler(string inputDirectory, int threads)
		{
			InputDirectory = inputDirectory;
			Threads = threads;
		}

		protected static string[] DefaultGlobalNumberVariables =
		{
			"LOCAL",
			"ABL",
			"RESULT",
			"SOURCE",
			"ARG",
			"GLOBAL",
			"IS",
			"CUP",
			"CDOWN",
			"UP",
			"DOWN",
			"DOWNBASE",
			"COUNT",
			"TCVAR",
			"CDFLAG",
			"ITEMPRICE",
			"RANDDATA",
			"LINECOUNT",
			"ISTIMEOUT",
			"__INT_MAX__",
			"__INT_MIN__",
			"RAND",
			"CHARANUM",
			"TALENT",
			"FLAG",
			"TFLAG",
			"CFLAG",
			"MASTER",
			"BASE",
			"MAXBASE",
			"PALAM",
			"TEQUIP",
			"EQUIP",
			"DAY",
			"MARK",
			"PALAMLV",
			"TARGET",
			"PLAYER",
			"NOWEX",
			"EX",
			"STAIN",
			"EXP",
			"ASSIPLAY",
			"ASSI",
			"ITEM",
			"EXPLV",
			"TIME",
			"MONEY",
			"SELECTCOM",
			"PREVCOM",
			"NEXTCOM",
			"DITEMTYPE",
			"NO",
			"RELATION",
			"JUEL",
			"GOTJUEL",
			"EJAC",
			"BOUGHT",
			"ITEMSALES",
			"FORWARD", //special casing for SORTCHARA
			"BACK", //special casing for SORTCHARA
			"LEFT", //special casing for __FORMAT
			"RIGHT", //special casing for __FORMAT
			"DEBUG_MODE",
			"NOITEM",
		};

		protected static string[] DefaultGlobalStringVariables =
		{
			"LOCALS",
			"ARGS",
			"RESULTS",
			"CSTR",
			"GLOBALS",
			"NICKNAME",
			"MASTERNAME",
			"NAME",
			"TRAINNAME",
			"BASENAME",
			"EQUIPNAME",
			"TEQUIPNAME",
			"STAINNAME",
			"EXNAME",
			"SOURCENAME",
			"CALLNAME",
			"FLAGNAME",
			"TFLAGNAME",
			"CFLAGNAME",
			"TCVARNAME",
			"STRNAME",
			"TSTRNAME",
			"CSTRNAME",
			"SAVESTRNAME",
			"CDFLAGNAME",
			"GLOBALNAME",
			"GLOBALSNAME",
			"GAMEBASE_AUTHOR",
			"GAMEBASE_INFO",
			"GAMEBASE_YEAR",
			"GAMEBASE_TITLE",
			"GAMEBASE_GAMECODE",
			"GAMEBASE_VERSION",
			"GAMEBASE_ALLOWVERSION",
			"GAMEBASE_DEFAULTCHARA",
			"GAMEBASE_NOITEM",
			"WINDOW_TITLE",
			"MONEYLABEL",
			"DRAWLINESTR",
			"LASTLOAD_VERSION",
			"LASTLOAD_NO",
			"LASTLOAD_TEXT",
			"SAVEDATA_TEXT",
			"TSTR",
			"STR",
			"SAVESTR",
			"ABLNAME",
			"MARKNAME",
			"TALENTNAME",
			"ITEMNAME",
			"PALAMNAME",
			"EXPNAME",
		};

		public void Compile(string outputDirectory)
		{
			var globals = new VariableDictionary();

			foreach (string stringVariable in DefaultGlobalStringVariables)
				globals.Add(stringVariable, "");

			foreach (string numberVariable in DefaultGlobalNumberVariables)
				globals.Add(numberVariable, 0);

			foreach (char c in "ABCDEFGHIJKLMNOPQRSTUVWXYZ")
				globals[c.ToString()] = "";

			var funcs = new List<FunctionDefinition>
			{
				new FunctionDefinition("ABS", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SQRT", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SIGN", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVCALLNAME", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOFULL", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOINT", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("RAND", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("VARSIZE", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOSTR", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0], "") }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOUPPER", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOLOWER", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOHALF", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOFULL", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("TOUPPER", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("ISNUMERIC", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("LOG10", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("ESCAPE", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("STRCOUNT", new[] { new FunctionParameter("input", new string[0]), new FunctionParameter("match", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("MIN", new[] { new FunctionParameter("a", new string[0], isArrayParameter: true), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("MAX", new[] { new FunctionParameter("a", new string[0], isArrayParameter: true), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("POWER", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETPALAMLV", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETBIT", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETEXPLV", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("UNICODE", new[] { new FunctionParameter("a", new string[0]) }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("MATCH", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]), new FunctionParameter("c", new string[0], "a"), new FunctionParameter("d", new string[0], "f"), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("INRANGE", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("b", new string[0]), new FunctionParameter("c", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("HE_SHE", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SUBSTRING", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("STRLENS", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVNAME", new[] { new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVNAME", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVBASE", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVTALENT", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVCFLAG", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVRELATION", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVCSTR", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVEXP", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CSVABL", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETNUM", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("FINDCHARA", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("LIMIT", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SUMCARRAY", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("MAXCARRAY", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SUBSTRINGU", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("FINDELEMENT", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0]),new FunctionParameter("a", new string[0]),new FunctionParameter("a", new string[0]), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GROUPMATCH", new[] { new FunctionParameter("a", new string[0]), new FunctionParameter("a", new string[0], isArrayParameter: true), }, new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETTIME", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("SAVENOS", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETCOLOR", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("CURRENTREDRAW", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETFONT", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
				new FunctionDefinition("GETMILLISECOND", new FunctionParameter[0], new FunctionVariable[0], true, "_GLOBAL", new Marker()),
			};

			var stringStatements = new List<Keyword>
			{
				new Keyword("DRAWLINEFORM", true, true),
				new Keyword("PRINTFORML", true, true),
				new Keyword("DATAFORM", true, true),
				new Keyword("PRINTFORMD", true, true),
				new Keyword("PRINTFORMDL", true, true),
				new Keyword("PRINTFORMW", true, true),
				new Keyword("PRINTFORMDW", true, true),
				new Keyword("PRINTFORMC", true, true),
				new Keyword("PRINTFORMLC", true, true),
				new Keyword("PRINTFORM", true, true),
				new Keyword("PRINTPLAINFORM", true, true),
				new Keyword("DEBUGPRINTFORM", true, true),
				new Keyword("DEBUGPRINTFORML", true, true),
				new Keyword("THROW", true, true),

				new Keyword("PRINT", true),
				new Keyword("PRINTD", true),
				new Keyword("PRINTDW", true),
				new Keyword("PRINTDL", true),
				new Keyword("PRINTW", true),
				new Keyword("PRINTV", true),
				new Keyword("PRINTL", true),
				new Keyword("PRINTLC", true),
				new Keyword("PRINTC", true),
				new Keyword("ALIGNMENT", true),
				new Keyword("CALL", true, true),
				new Keyword("CUSTOMDRAWLINE", true),
				new Keyword("GOTO", true),
				new Keyword("DEBUGPRINTL", true),
				new Keyword("REUSELASTLINE", true),
				new Keyword("PRINTPLAIN", true),
				new Keyword("PRINT_TRAIN_NAME", true),
				new Keyword("PRINT_STR", true),
			};

			string csvPath = Path.Combine(InputDirectory, "CSV");
			string erbPath = Path.Combine(InputDirectory, "ERB");

			var csvDefinition = new CSVDefinition();

			Console.WriteLine("Preprocessing CSV files...");


			foreach (var file in Directory.EnumerateFiles(csvPath, "*.csv", SearchOption.AllDirectories))
			{
				string path = Path.GetFileNameWithoutExtension(file);

				if (path.EndsWith("_TR"))
					continue;

				Preprocessor.ProcessCSV(csvDefinition, path, File.ReadLines(file, Encoding.UTF8));
			}

			Console.WriteLine("Preprocessing header files...");

			ConcurrentBag<FunctionVariable> preprocessedConstants = new ConcurrentBag<FunctionVariable>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(erbPath, "*.erh", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(erbPath, "*.erh", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				try
				{
					foreach (var variable in Preprocessor.PreprocessHeaderFile(File.ReadAllText(file)))
						preprocessedConstants.Add(variable);
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal pre-process lexer error [{ex}]", new Marker()), Path.GetFileName(file)));
				}
#if DEBUG
			}
#else
			});
#endif

			Console.WriteLine("Preprocessing functions...");

			ConcurrentDictionary<FunctionDefinition, string> preprocessedFunctions = new ConcurrentDictionary<FunctionDefinition, string>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(erbPath, "*.erb", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(erbPath, "*.erb", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				try
				{
					foreach (var kv in Preprocessor.PreprocessCodeFile(File.ReadAllText(file), Path.GetFileName(file), preprocessedConstants.ToArray()))
						preprocessedFunctions[kv.Key] = kv.Value;
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal pre-process lexer error [{ex}]", new Marker()), Path.GetFileName(file)));
				}
#if DEBUG
			}
#else
			});
#endif

			DeclaredFunctions = preprocessedFunctions.ToDictionary(kv => kv.Key, kv => kv.Value);

			funcs.AddRange(DeclaredFunctions.Keys.Where(x => x.IsReturnFunction));
			var procedures = DeclaredFunctions.Keys.Where(x => !x.IsReturnFunction).ToList();


			Console.WriteLine("Compiling functions...");

#if DEBUG
			foreach (var kv in DeclaredFunctions)
#else
			Parallel.ForEach(DeclaredFunctions, new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, kv =>
#endif
			{
				try
				{
					var locals = new VariableDictionary();
					foreach (var param in kv.Key.Variables)
						locals[param.Name] = param.CalculatedValue;

					Parser parser = new Parser(kv.Value, kv.Key, funcs, procedures, globals, locals, stringStatements, csvDefinition, preprocessedConstants.ToArray());

					var nodes = parser.Parse(out var localErrors, out var localWarnings);

					lock (Errors)
						Errors.AddRange(localErrors.Select(x => new Tuple<ParserError, string>(x, $"{kv.Key.Filename} - {kv.Key.Name}")));

					lock (Warnings)
						Warnings.AddRange(localWarnings.Select(x => new Tuple<ParserError, string>(x, $"{kv.Key.Filename} - {kv.Key.Name}")));
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal post-process lexer error [{ex}]", new Marker()), $"{kv.Key.Filename} - {kv.Key.Name}"));
				}
#if DEBUG
			}
#else
			});
#endif

			long fileSize = 0;
			int fileCount = 0;

			foreach (var file in Directory.EnumerateFiles(InputDirectory, "*.erb", SearchOption.AllDirectories))
			{
				var info = new FileInfo(file);
				fileSize += info.Length;
				fileCount++;
			}

			var htmlWriter = new HTMLWriter
			{
				Errors = Errors.OrderBy(x => x.Item2).ToList(),
				Warnings = Warnings.OrderBy(x => x.Item2).ToList(),
				FunctionCount = DeclaredFunctions.Count,
				TotalFileSize = fileSize,
				TotalFileCount = fileCount
			};

			using (var stream = File.Create(Path.Combine(outputDirectory, "report.html")))
				htmlWriter.WriteReportToFile(stream);
		}
	}
}