NTERA/NTERA.Engine/Runtime/JITCompiler.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Runtime;
using System.Text;
using System.Threading.Tasks;
using NTERA.Core;
using NTERA.Engine.Compiler;
using NTERA.Engine.Runtime.Resources;

namespace NTERA.Engine.Runtime
{
	public class JITCompiler : IExecutionProvider
	{
		public ICollection<FunctionDefinition> DefinedProcedures { get; protected set; }
		public ICollection<FunctionDefinition> DefinedFunctions { get; protected set; }
		public ICollection<FunctionVariable> DefinedConstants { get; protected set; }
		public CSVDefinition CSVDefinition { get; protected set; }

		protected Dictionary<string, ImageDefinition> ImageDefinitions { get; set; }

		protected Dictionary<FunctionDefinition, string> ProcedureFiles { get; set; }
		public Dictionary<FunctionDefinition, ExecutionNode[]> CompiledProcedures { get; protected set; } = new Dictionary<FunctionDefinition, ExecutionNode[]>();

		public string InputDirectory { get; }
		protected string CSVPath { get; set; }
		protected string ERBPath { get; set; }
		protected string ResourcePath { get; set; }

		public static int Threads = 4;

		public JITCompiler(string path)
		{
			InputDirectory = path;
		}

		public void Initialize(IConsole console)
		{
			Stopwatch stopwatch = new Stopwatch();
			stopwatch.Start();

			CSVPath = Path.Combine(InputDirectory, "CSV");
			ERBPath = Path.Combine(InputDirectory, "ERB");
			ResourcePath = Path.Combine(InputDirectory, "resources");

			CSVDefinition = new CSVDefinition();
			ImageDefinitions = new Dictionary<string, ImageDefinition>();

			if (!Directory.Exists(InputDirectory) || !Directory.Exists(CSVPath) || !Directory.Exists(ERBPath)) {
				console.PrintError($"{InputDirectory} does not appear to be an era game.  Expecting to find {InputDirectory}/CSV and /ERB and /resources.\nPlease set current working directory or the ERA environment variable to the era folder.");
				return;
			}

			console.PrintSystemLine("Preprocessing CSV files...");


			foreach (var file in Directory.EnumerateFiles(CSVPath, "*.csv", SearchOption.AllDirectories))
			{
				string localName = Path.GetFileNameWithoutExtension(file);

				if (localName == null || localName.EndsWith("_TR"))
					continue;

				Preprocessor.ProcessCSV(CSVDefinition, localName, File.ReadLines(file, Encoding.UTF8));
			}


			foreach (var file in Directory.EnumerateFiles(ResourcePath, "*.csv", SearchOption.TopDirectoryOnly))
			{
				foreach (var line in Preprocessor.SplitCSV(File.ReadLines(file)))
				{
					if (line.Count != 2 && line.Count != 6)
						throw new ParserException($"Unable to parse resource CSV file '{Path.GetFileName(file)}'");

					ImageDefinition definition = new ImageDefinition
					{
						Name = line[0],
						Filename = line[1],
						Dimensions = null
					};

					if (line.Count == 6)
					{
						definition.Dimensions = new Rectangle(int.Parse(line[2]), int.Parse(line[3]), int.Parse(line[4]), int.Parse(line[5]));
					}

					ImageDefinitions[definition.Name] = definition;
				}
			}

			console.PrintSystemLine("Preprocessing header files...");

			ConcurrentBag<FunctionVariable> preprocessedConstants = new ConcurrentBag<FunctionVariable>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(ERBPath, "*.erh", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(ERBPath, "*.erh", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				foreach (var variable in Preprocessor.PreprocessHeaderFile(File.ReadAllText(file)))
					preprocessedConstants.Add(variable);
#if DEBUG
			}
#else
			});
#endif

			DefinedConstants = preprocessedConstants.ToArray();

			console.PrintSystemLine("Preprocessing functions...");

			ProcedureFiles = new Dictionary<FunctionDefinition, string>();
			ConcurrentDictionary<FunctionDefinition, string> preprocessedFunctions = new ConcurrentDictionary<FunctionDefinition, string>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(ERBPath, "*.erb", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(ERBPath, "*.erb", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				foreach (var kv in Preprocessor.PreprocessCodeFile(File.ReadAllText(file), Path.GetFileName(file), DefinedConstants))
				{
					preprocessedFunctions[kv.Key] = kv.Value;
					ProcedureFiles[kv.Key] = file;
				}
#if DEBUG
			}
#else
			});
#endif
			DefinedProcedures = preprocessedFunctions.Select(kv => kv.Key).ToArray();

			var declaredFunctions = BaseDefinitions.DefaultGlobalFunctions.ToList();
			declaredFunctions.AddRange(DefinedProcedures.Where(x => x.IsReturnFunction));
			DefinedFunctions = declaredFunctions;

			console.PrintSystemLine("Compacting memory...");

			GCSettings.LargeObjectHeapCompactionMode = GCLargeObjectHeapCompactionMode.CompactOnce;
			GC.Collect();

			stopwatch.Stop();
			console.PrintSystemLine($"Completed initialization in {stopwatch.ElapsedMilliseconds}ms");
		}

		public IEnumerable<ExecutionNode> GetExecutionNodes(FunctionDefinition function)
		{
			if (CompiledProcedures.ContainsKey(function))
				return CompiledProcedures[function];

			string filename = ProcedureFiles[function];
			var preprocessed = Preprocessor.PreprocessCodeFile(File.ReadAllText(filename), Path.GetFileName(filename), DefinedConstants);

			Parser parser = new Parser(preprocessed.Single(x => x.Key.Name == function.Name).Value, function, DefinedFunctions, DefinedProcedures.Where(x => !x.IsReturnFunction).ToArray(), BaseDefinitions.DefaultGlobalVariables, function.Variables, BaseDefinitions.DefaultKeywords, CSVDefinition, DefinedConstants);

			var nodes = parser.Parse(out var localErrors, out var localWarnings)?.ToArray();

			if (localErrors.Count > 0)
				throw new ParserException($"Failed to compile '{function.Name}'");

			CompiledProcedures.Add(function, nodes);

			return nodes;
		}

		protected Dictionary<string, Bitmap> BitmapCache = new Dictionary<string, Bitmap>();

		public Bitmap GetImage(string imageName, out ImageDefinition definition)
		{
			definition = ImageDefinitions[imageName];

			string filename = Path.Combine(ResourcePath, definition.Filename);

			if (!BitmapCache.TryGetValue(filename, out var bitmap))
			{
				bitmap = new Bitmap(filename);
				BitmapCache.Add(filename, bitmap);
			}

			return bitmap;
		}
	}
}