Implement preliminary preprocessing

NTERA.Interpreter/Engine.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections.Generic;
 using System.Drawing;
 using System.IO;
 using NTERA.Core;
@@ -12,21 +13,28 @@ namespace NTERA.Interpreter
 
         public string EntrypointPath { get; protected set; }
 
-        public Interpreter Interpreter { get; protected set; }
+        public Dictionary<string, Interpreter> Procedures = new Dictionary<string, Interpreter>();
+
+        public Stack<Interpreter> CallStack = new Stack<Interpreter>();
+
+	    public Interpreter CurrentParser => CallStack.Count > 0 ? CallStack.Peek() : null;
 
 		public bool Initialize(IConsole console)
 		{
             Console = console;
 
             EntrypointPath = @"M:\era\eraSemifullTest\erb\SYSTEM_TITLE.ERB"; //@"M:\era\eraSemifullTest\erb\TWTITLE.txt";
-            Interpreter = new Interpreter(console, File.ReadAllText(EntrypointPath));
+
+		    Preprocess(File.ReadAllText(EntrypointPath));
+
+		    CallStack.Push(new Interpreter(console, File.ReadAllText(EntrypointPath)));
 
             return true;
         }
 
 		public void Start()
 		{
-            Interpreter.Exec();
+            CurrentParser.Exec();
 		}
 
 		public void InputString(string input)
@@ -49,5 +57,89 @@ namespace NTERA.Interpreter
 		    var bitmap = new Bitmap(@"M:\era\eraSemifullTest\resources\bbb.png");
             return new CroppedImage(name, bitmap, new Rectangle(Point.Empty, bitmap.Size), false);
 		}
+
+	    public Dictionary<string, Interpreter> Preprocess(string contents)
+	    {
+            Dictionary<string, Interpreter> procs = new Dictionary<string, Interpreter>();
+
+	        Lexer lexer = new Lexer(contents);
+
+	        Marker startMarker = lexer.TokenMarker;
+	        string currentProc = null;
+            VariableDictionary locals = new VariableDictionary();
+
+	        void Commit()
+	        {
+	            if (currentProc != null)
+	            {
+	                string procBody = contents.Substring(startMarker.Pointer,
+	                    lexer.TokenMarker.Pointer - startMarker.Pointer);
+
+	                var interpreter = new Interpreter(Console, procBody)
+	                {
+	                    Variables = locals
+	                };
+
+	                procs.Add(currentProc, interpreter);
+	            }
+            }
+
+	        using (var enumerator = lexer.GetEnumerator())
+	            do
+	            {
+	                if (enumerator.Current == Token.Function)
+	                {
+	                    Commit();
+
+	                    enumerator.MoveNext();
+	                    if (enumerator.Current != Token.Identifer)
+	                        throw new InvalidOperationException();
+
+	                    currentProc = lexer.Identifer;
+	                }
+	                else if (enumerator.Current == Token.Sharp)
+	                {
+	                    enumerator.MoveNext();
+
+	                    switch (enumerator.Current)
+	                    {
+	                        case Token.Dim:
+	                        {
+	                            bool isString = enumerator.Current != Token.Dim;
+
+	                            enumerator.MoveNext();
+
+	                            while (enumerator.MoveNext() && lexer.Identifer == "CONST")
+	                            {
+	                            }
+
+	                            string variable = lexer.Identifer;
+
+	                            enumerator.MoveNext();
+	                            enumerator.MoveNext();
+
+	                            if (isString)
+	                                lexer.Type = LexerType.String;
+
+	                            locals[variable] = lexer.Expression();
+
+	                            lexer.Type = LexerType.Both;
+
+	                            break;
+	                        }
+	                        case Token.ReturnFunction:
+	                        {
+	                            break;
+	                        }
+	                    }
+	                }
+
+                    
+	            } while (enumerator.MoveNext());
+
+            Commit();
+
+	        return procs;
+	    }
 	}
 }

NTERA.Interpreter/Interpreter.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using NTERA.Core;
 
 namespace NTERA.Interpreter
@@ -11,7 +10,7 @@ namespace NTERA.Interpreter
         private Token previousToken;
         private Token CurrentToken => Tokens.Current;
 
-        public VariableDictionary Variables { get; } = new VariableDictionary();
+        public VariableDictionary Variables { get; internal set; } = new VariableDictionary();
         protected Dictionary<string, Marker> Loops { get; } = new Dictionary<string, Marker>();
 
         public delegate Value BasicFunction(List<Value> args);
@@ -33,7 +32,7 @@ namespace NTERA.Interpreter
 
         private void Error(string text)
         {
-            throw new Exception(text + " at line: " + LineMarker.Line);
+            throw new ParserException(text, LineMarker);
         }
 
         protected bool TryAssertToken(Token tok, bool pullNext = true)
@@ -56,9 +55,7 @@ namespace NTERA.Interpreter
         {
             exit = false;
 
-            Tokens = Lexer.GetTokens().GetEnumerator();
-
-            GetNextToken();
+            Tokens = Lexer.GetEnumerator();
 
             while (!exit)
                 Line();
@@ -157,150 +154,24 @@ namespace NTERA.Interpreter
             }
         }
 
-        private static readonly Dictionary<Token, int> OrderOfOps = new Dictionary<Token, int>
-        {
-            { Token.Or, 0 }, { Token.And, 0 },
-            { Token.Equal, 1 }, { Token.NotEqual, 1 },
-            { Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 },  { Token.MoreEqual, 1 },
-            { Token.Plus, 2 }, { Token.Minus, 2 },
-            { Token.Asterisk, 3 }, {Token.Slash, 3 },
-            { Token.Caret, 4 }
-        };
-
         private Value RealExpression()
         {
-            return Expression(Lexer, Tokens);
+            return Lexer.Expression(this);
         }
 
         private Value RealExpression(string input)
         {
-            return Expression(new Lexer(input));
+            return new Lexer(input).Expression(this);
         }
 
         private Value StringExpression()
         {
             Lexer.Type = LexerType.String;
             GetNextToken();
-            var result = Expression(Lexer, Tokens, LexerType.String);
+            var result = Lexer.Expression(this);
             Lexer.Type = LexerType.Real;
 
             return result;
         }
-
-        private Value Expression(Lexer lexer, IEnumerator<Token> enumerator = null, LexerType type = LexerType.Real)
-        {
-            Stack<Value> stack = new Stack<Value>();
-            Stack<Token> operators = new Stack<Token>();
-
-            if (enumerator == null)
-            {
-                enumerator = lexer.GetTokens().GetEnumerator();
-                enumerator.MoveNext();
-            }
-
-            void Operation(Token token)
-            {
-                Value b = stack.Pop();
-                Value a = stack.Pop();
-                Value result = a.Operate(b, token);
-                stack.Push(result);
-            }
-
-            int i = 0;
-            while (true)
-            {
-                if (enumerator.Current == Token.Value)
-                {
-                    stack.Push(lexer.Value);
-                }
-                else if (enumerator.Current == Token.Identifer)
-                {
-                    if (Variables.ContainsKey(lexer.Identifer))
-                    {
-                        string varName = lexer.Identifer;
-                        int index = 0;
-                        enumerator.MoveNext();
-                        if (enumerator.Current == Token.Colon)
-                        {
-                            enumerator.MoveNext();
-                            index = (int)Expression(lexer, enumerator).Real;
-                        }
-
-                        stack.Push(Variables[varName, index]);
-
-                        i++;
-                        continue;
-                    }
-                    else if (FunctionDictionary.ContainsKey(lexer.Identifer))
-                    {
-                        string name = lexer.Identifer;
-                        List<Value> args = new List<Value>();
-                        enumerator.MoveNext();
-
-                        if (enumerator.Current != Token.LParen)
-                            Error($"Was expecting [LParen] got [{enumerator.Current}]");
-
-                        while (enumerator.MoveNext() && enumerator.Current != Token.RParen)
-                        {
-                            args.Add(Expression(lexer, enumerator, type));
-                            if (enumerator.Current != Token.Comma)
-                                break;
-                        }
-
-                        stack.Push(FunctionDictionary[name](args));
-                    }
-                    else
-                    {
-                        if (type == LexerType.String)
-                            stack.Push(lexer.Identifer);
-                        else
-                            Error("Undeclared variable " + lexer.Identifer);
-                    }
-                }
-                else if (enumerator.Current == Token.LParen)
-                {
-                    enumerator.MoveNext();
-                    stack.Push(Expression(lexer, enumerator, type));
-
-                    if (enumerator.Current != Token.RParen)
-                        Error($"Was expecting [LParen] got [{enumerator.Current}]");
-                }
-                else if (type == LexerType.Real && enumerator.Current.IsArithmetic()
-                         && enumerator.Current.IsUnary() && (i == 0 || previousToken == Token.LParen))
-                {
-                    stack.Push(0);
-                    operators.Push(enumerator.Current);
-                }
-                else if (type == LexerType.String && enumerator.Current.IsStringOp()
-                        || type == LexerType.Real && enumerator.Current.IsArithmetic())
-                {
-                    while (operators.Count > 0 && OrderOfOps[enumerator.Current] <= OrderOfOps[operators.Peek()])
-                        Operation(operators.Pop());
-                    operators.Push(enumerator.Current);
-                }
-                else
-                {
-                    if (i == 0)
-                    {
-                        if (type == LexerType.String)
-                            stack.Push("");
-                        else
-                            Error("Empty expression");
-                    }
-
-                    break;
-                }
-
-                i++;
-                enumerator.MoveNext();
-            }
-
-            while (operators.Count > 0)
-                Operation(operators.Pop());
-
-            return type == LexerType.String 
-                ? stack.Aggregate((a, b) => b.String + a.String) 
-                : stack.Pop();
-        }
     }
 }

NTERA.Interpreter/Interpreter/Keywords.cs

@@ -36,7 +36,7 @@ namespace NTERA.Interpreter
         [KeywordMethod(Token.PrintL)]
         private void PrintL()
         {
-            Console.PrintSingleLine(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)));
+            Console.PrintSingleLine(ParseFormat(new Lexer(Lexer.Value, LexerType.String).Expression(this)));
 
             GetNextToken();
         }
@@ -46,7 +46,7 @@ namespace NTERA.Interpreter
         {
             AssertToken(Token.Value, false);
 
-            Console.PrintHtml(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)), true);
+            Console.PrintHtml(ParseFormat(new Lexer(Lexer.Value, LexerType.String).Expression(this)), true);
 
             GetNextToken();
         }
@@ -75,7 +75,7 @@ namespace NTERA.Interpreter
 
         private string ParseFormat(string rawInput)
         {
-            var realEvaluator = new MatchEvaluator(match => Expression(new Lexer(match.Groups[1].Value)).ToString());
+            var realEvaluator = new MatchEvaluator(match => new Lexer(match.Groups[1].Value).Expression(this));
 
             string reals = RealFormatRegex.Replace(rawInput, realEvaluator);
 
@@ -328,7 +328,7 @@ namespace NTERA.Interpreter
 
         #region Global
 
-        [KeywordMethod(Token.Global)]
+        [KeywordMethod(Token.Sharp)]
         void Global()
         {
             if (TryAssertToken(Token.Dim, false))

NTERA.Interpreter/Lexer.cs

@@ -1,16 +1,20 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Linq;
 using System.Text;
 
 namespace NTERA.Interpreter
 {
-    public class Lexer
+    public class Lexer : IEnumerable<Token>
     {
         private readonly string source;
         private Marker sourceMarker;
         private char currentChar;
 
+        private IEnumerator<Token> currentEnumerator;
+
         private LexerType _type;
         public LexerType Type
         {
@@ -33,6 +37,9 @@ namespace NTERA.Interpreter
 
             source = input;
             sourceMarker = new Marker(-1, 1, 0);
+
+            currentEnumerator = GetTokens();
+            currentEnumerator.MoveNext();
         }
 
         public void GoTo(Marker marker)
@@ -195,8 +202,10 @@ namespace NTERA.Interpreter
             return Token.Unknown;
         }
 
-        public IEnumerable<Token> GetTokens()
+        private IEnumerator<Token> GetTokens()
         {
+            sourceMarker = new Marker(-1, 1, 0);
+
             while (true)
             {
                 while (IsWhitespace(GetNextChar()) && Type != LexerType.String || currentChar == '\r') { }
@@ -279,5 +288,140 @@ namespace NTERA.Interpreter
                 yield return Token.Identifer;
             }
         }
+
+        public IEnumerator<Token> GetEnumerator()
+        {
+            return currentEnumerator;
+        }
+
+        IEnumerator IEnumerable.GetEnumerator()
+        {
+            return GetEnumerator();
+        }
+
+        private static readonly Dictionary<Token, int> OrderOfOps = new Dictionary<Token, int>
+        {
+            { Token.Or, 0 }, { Token.And, 0 },
+            { Token.Equal, 1 }, { Token.NotEqual, 1 },
+            { Token.Less, 1 }, { Token.More, 1 }, { Token.LessEqual, 1 },  { Token.MoreEqual, 1 },
+            { Token.Plus, 2 }, { Token.Minus, 2 },
+            { Token.Asterisk, 3 }, {Token.Slash, 3 },
+            { Token.Caret, 4 }
+        };
+
+        public Value Expression(Interpreter context = null)
+        {
+            Stack<Value> stack = new Stack<Value>();
+            Stack<Token> operators = new Stack<Token>();
+
+            void Operation(Token token)
+            {
+                Value b = stack.Pop();
+                Value a = stack.Pop();
+                Value result = a.Operate(b, token);
+                stack.Push(result);
+            }
+
+            int i = 0;
+            while (true)
+            {
+                if (currentEnumerator.Current == Token.Value)
+                {
+                    stack.Push(Value);
+                }
+                else if (currentEnumerator.Current == Token.Identifer)
+                {
+                    if (context != null)
+                    {
+                        if (context.Variables.ContainsKey(Identifer))
+                        {
+                            string varName = Identifer;
+                            int index = 0;
+                            currentEnumerator.MoveNext();
+                            if (currentEnumerator.Current == Token.Colon)
+                            {
+                                currentEnumerator.MoveNext();
+                                index = (int)Expression(context).Real;
+                            }
+
+                            stack.Push(context.Variables[varName, index]);
+
+                            i++;
+                            continue;
+                        }
+
+                        if (context.FunctionDictionary.ContainsKey(Identifer))
+                        {
+                            string name = Identifer;
+                            List<Value> args = new List<Value>();
+                            currentEnumerator.MoveNext();
+
+                            if (currentEnumerator.Current != Token.LParen)
+                                throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);
+
+                            while (currentEnumerator.MoveNext() && currentEnumerator.Current != Token.RParen)
+                            {
+                                args.Add(Expression(context));
+                                if (currentEnumerator.Current != Token.Comma)
+                                    break;
+                            }
+
+                            stack.Push(context.FunctionDictionary[name](args));
+                            currentEnumerator.MoveNext();
+                            i++;
+                            continue;
+                        }
+                    }
+                    
+                    if (Type == LexerType.String)
+                        stack.Push(Identifer);
+                    else
+                        throw new ParserException("Undeclared variable " + Identifer, TokenMarker);
+                }
+                else if (currentEnumerator.Current == Token.LParen)
+                {
+                    currentEnumerator.MoveNext();
+                    stack.Push(Expression());
+
+                    if (currentEnumerator.Current != Token.RParen)
+                        throw new ParserException($"Was expecting [LParen] got [{currentEnumerator.Current}]", TokenMarker);
+                }
+                else if (Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic()
+                         && currentEnumerator.Current.IsUnary() && (i == 0)) // || previousToken == Token.LParen))
+                {
+                    stack.Push(0);
+                    operators.Push(currentEnumerator.Current);
+                }
+                else if (Type == LexerType.String && currentEnumerator.Current.IsStringOp()
+                        || Type.HasFlag(LexerType.Real) && currentEnumerator.Current.IsArithmetic())
+                {
+                    while (operators.Count > 0 && OrderOfOps[currentEnumerator.Current] <= OrderOfOps[operators.Peek()])
+                        Operation(operators.Pop());
+                    operators.Push(currentEnumerator.Current);
+                }
+                else
+                {
+                    if (i == 0)
+                    {
+                        if (Type == LexerType.String)
+                            stack.Push("");
+                        else
+                            throw new ParserException("Empty expression", TokenMarker);
+                    }
+
+                    break;
+                }
+
+                i++;
+                currentEnumerator.MoveNext();
+            }
+
+            while (operators.Count > 0)
+                Operation(operators.Pop());
+
+            return Type == LexerType.String
+                ? stack.Aggregate((a, b) => b.String + a.String)
+                : stack.Pop();
+        }
     }
 }

NTERA.Interpreter/Marker.cs

@@ -12,5 +12,10 @@
             Line = line;
             Column = column;
         }
+
+        public override string ToString()
+        {
+            return $"line {Line}, column {Column}";
+        }
     }
 }

NTERA.Interpreter/NTERA.Interpreter.csproj

@@ -45,6 +45,7 @@
     <Compile Include="Interpreter\Keywords.cs" />
     <Compile Include="Lexer.cs" />
     <Compile Include="Marker.cs" />
+    <Compile Include="ParserException.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="Token.cs" />
     <Compile Include="Utility.cs" />

NTERA.Interpreter/ParserException.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace NTERA.Interpreter
+{
+    public class ParserException : Exception
+    {
+        public ParserException(string message) : base(message)
+        {
+
+        }
+
+        public ParserException(string message, Marker marker) : base(message + " at " + marker)
+        {
+
+        }
+    }
+}

NTERA.Interpreter/Token.cs

@@ -8,7 +8,7 @@
         Value,
 
         [LexerCharacter('#', LexerType.Real)]
-        Global,
+        Sharp,
         [LexerCharacter('@', LexerType.Real)]
         Function,
 
@@ -16,6 +16,8 @@
         Dim,
         [LexerKeyword("CONST")]
         Const,
+        [LexerKeyword("FUNCTION")]
+        ReturnFunction,
 
         //Eralang print keywords 
         [LexerKeyword("DRAWLINE", false)]