NTERA/NTERA.Compiler/Compiler.cs

using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
using NTERA.Engine;
using NTERA.Engine.Compiler;
using NTERA.Engine.Runtime;

namespace NTERA.Compiler
{
	public class Compiler
	{
		public string InputDirectory { get; }

		public List<Tuple<ParserError, string>> Errors { get; } = new List<Tuple<ParserError, string>>();
		public List<Tuple<ParserError, string>> Warnings { get; } = new List<Tuple<ParserError, string>>();

		public Dictionary<FunctionDefinition, string> DeclaredProcedures = new Dictionary<FunctionDefinition, string>();

		public int Threads { get; set; }

		public Compiler(string inputDirectory, int threads)
		{
			InputDirectory = inputDirectory;
			Threads = threads;
		}

		public void Compile(string outputDirectory)
		{
			string csvPath = Path.Combine(InputDirectory, "CSV");
			string erbPath = Path.Combine(InputDirectory, "ERB");

			var csvDefinition = new CSVDefinition();

			Console.WriteLine("Preprocessing CSV files...");


			foreach (var file in Directory.EnumerateFiles(csvPath, "*.csv", SearchOption.AllDirectories))
			{
				string path = Path.GetFileNameWithoutExtension(file);

				if (path.EndsWith("_TR"))
					continue;

				Preprocessor.ProcessCSV(csvDefinition, path, File.ReadLines(file, Encoding.UTF8));
			}

			Console.WriteLine("Preprocessing header files...");

			ConcurrentBag<FunctionVariable> preprocessedConstants = new ConcurrentBag<FunctionVariable>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(erbPath, "*.erh", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(erbPath, "*.erh", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				try
				{
					foreach (var variable in Preprocessor.PreprocessHeaderFile(File.ReadAllText(file)))
						preprocessedConstants.Add(variable);
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal pre-process lexer error [{ex}]", new Marker()), Path.GetFileName(file)));
				}
#if DEBUG
			}
#else
			});
#endif

			Console.WriteLine("Preprocessing functions...");

			ConcurrentDictionary<FunctionDefinition, string> preprocessedFunctions = new ConcurrentDictionary<FunctionDefinition, string>();

#if DEBUG
			foreach (var file in Directory.EnumerateFiles(erbPath, "*.erb", SearchOption.AllDirectories))
#else
			Parallel.ForEach(Directory.EnumerateFiles(erbPath, "*.erb", SearchOption.AllDirectories), new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, file =>
#endif
			{
				try
				{
					foreach (var kv in Preprocessor.PreprocessCodeFile(File.ReadAllText(file), Path.GetFileName(file), preprocessedConstants.ToArray()))
						preprocessedFunctions[kv.Key] = kv.Value;
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal pre-process lexer error [{ex}]", new Marker()), Path.GetFileName(file)));
				}
#if DEBUG
			}
#else
			});
#endif

			DeclaredProcedures = preprocessedFunctions.ToDictionary(kv => kv.Key, kv => kv.Value);

			var declaredFunctions = BaseDefinitions.DefaultGlobalFunctions.ToList();
			declaredFunctions.AddRange(DeclaredProcedures.Keys.Where(x => x.IsReturnFunction));

			var procedures = DeclaredProcedures.Keys.Where(x => !x.IsReturnFunction).ToList();


			Console.WriteLine("Compiling functions...");

#if DEBUG
			foreach (var kv in DeclaredProcedures)
#else
			Parallel.ForEach(DeclaredProcedures, new ParallelOptions
			{
				MaxDegreeOfParallelism = Threads
			}, kv =>
#endif
			{
				try
				{
					Parser parser = new Parser(kv.Value, kv.Key, declaredFunctions, procedures, BaseDefinitions.DefaultGlobalVariables, kv.Key.Variables, BaseDefinitions.DefaultKeywords, csvDefinition, preprocessedConstants.ToArray());

					var nodes = parser.Parse(out var localErrors, out var localWarnings);

					using (var str = File.Create(Path.Combine(outputDirectory, $"{kv.Key.Name}.xml")))
					using (var xml = XmlWriter.Create(str, new XmlWriterSettings
					{
						Indent = true
					}))
					{
						new ExecutionNode
						{
							Type = "exec",
							Metadata =
							{
								["ast-version"] = "2",
									["name"] = kv.Key.Name,
									["filename"] = kv.Key.Filename,
									["position"] = kv.Key.Position.ToString()
							},
							SubNodes = nodes.ToArray()
						}.WriteXml(xml);
					}

					lock (Errors)
						Errors.AddRange(localErrors.Select(x => new Tuple<ParserError, string>(x, $"{kv.Key.Filename} - {kv.Key.Name}")));

					lock (Warnings)
						Warnings.AddRange(localWarnings.Select(x => new Tuple<ParserError, string>(x, $"{kv.Key.Filename} - {kv.Key.Name}")));
				}
				catch (Exception ex)
				{
					lock (Errors)
						Errors.Add(new Tuple<ParserError, string>(new ParserError($"Internal post-process lexer error [{ex}]", new Marker()), $"{kv.Key.Filename} - {kv.Key.Name}"));
				}
#if DEBUG
			}
#else
			});
#endif

			long fileSize = 0;
			int fileCount = 0;

			foreach (var file in Directory.EnumerateFiles(InputDirectory, "*.erb", SearchOption.AllDirectories))
			{
				var info = new FileInfo(file);
				fileSize += info.Length;
				fileCount++;
			}

			var htmlWriter = new HTMLWriter
			{
				Errors = Errors.OrderBy(x => x.Item2).ToList(),
				Warnings = Warnings.OrderBy(x => x.Item2).ToList(),
				FunctionCount = DeclaredProcedures.Count,
				TotalFileSize = fileSize,
				TotalFileCount = fileCount
			};

			using (var stream = File.Create(Path.Combine(outputDirectory, "report.html")))
				htmlWriter.WriteReportToFile(stream);
		}
	}
}