Partially rework lexer

NTERA.Interpreter/Attributes.cs

@@ -1,8 +1,4 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 
 namespace NTERA.Interpreter
 {

NTERA.Interpreter/Interpreter.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Linq;
 using System.Reflection;
+using System.Text.RegularExpressions;
 using NTERA.Core;
 
 namespace NTERA.Interpreter
@@ -61,9 +62,14 @@ namespace NTERA.Interpreter
                 Error("Expect " + tok + " got " + lastToken);
         }
 
+        private IEnumerator<Token> tokens;
+
         public void Exec()
         {
             exit = false;
+
+            tokens = lex.GetTokens().GetEnumerator();
+
             GetNextToken();
             while (!exit) Line();
         }
@@ -71,7 +77,9 @@ namespace NTERA.Interpreter
         Token GetNextToken()
         {
             prevToken = lastToken;
-            lastToken = lex.GetToken();
+
+            tokens.MoveNext();
+            lastToken = tokens.Current;
 
             if (lastToken == Token.EOF && prevToken == Token.EOF)
                 Error("Unexpected end of file");
@@ -255,6 +263,18 @@ namespace NTERA.Interpreter
             console.PrintSingleLine(Expr().ToString());
         }
 
+        private static Regex formRegex = new Regex("{(.*?)}");
+
+        [TargetKeyword(Token.PrintFormL)]
+        void PrintFormL()
+        {
+            string rawString = Expr().ToString();
+
+            var evaluator = new MatchEvaluator((match) => { return vars[match.Groups[1].Value].ToString(); });
+
+            console.PrintSingleLine(formRegex.Replace(rawString, evaluator));
+        }
+
         [TargetKeyword(Token.DrawLine)]
         void DrawLine()
         {
@@ -264,7 +284,7 @@ namespace NTERA.Interpreter
         [TargetKeyword(Token.DrawLineForm)]
         void DrawLineForm()
         {
-            console.printCustomBar(Expr().ToString());
+            console.printCustomBar(Expr().ToString().Trim());
         }
 
         [TargetKeyword(Token.If)]
@@ -411,6 +431,24 @@ namespace NTERA.Interpreter
             lastToken = Token.NewLine;
         }
 
+        [TargetKeyword(Token.Times)]
+        private void Times()
+        {
+            Match(Token.Identifer);
+
+            string var = lex.Identifer;
+
+            GetNextToken();
+            Match(Token.Comma);
+
+            GetNextToken();
+            var arg2 = Expr();
+
+            vars[var] = vars[var].BinOp(arg2, Token.Asterisk);
+            
+            Match(Token.NewLine);
+        }
+
         #endregion
 
         void Input()

NTERA.Interpreter/Lexer.cs

@@ -1,4 +1,6 @@
 using System;
+using System.Collections.Generic;
+using System.Text;
 
 namespace NTERA.Interpreter
 {
@@ -16,8 +18,7 @@ namespace NTERA.Interpreter
         public Lexer(string input)
         {
             source = input;
-            sourceMarker = new Marker(0, 1, 1);
-            lastChar = source[0];
+            sourceMarker = new Marker(-1, 1, 0);
         }
 
         public void GoTo(Marker marker)
@@ -41,121 +42,192 @@ namespace NTERA.Interpreter
             return lastChar;
         }
 
-        public Token GetToken()
+        private readonly Dictionary<string, Token> TokenDictionary = new Dictionary<string, Token>(StringComparer.InvariantCultureIgnoreCase)
         {
-            while (lastChar == ' ' || lastChar == '\t' || lastChar == '\r')
-                GetChar();
+            //["PRINT"] = Token.Print,
+            //["PRINTL"] = Token.PrintL,
+            ["DRAWLINE"] = Token.DrawLine,
+            //["DRAWLINEFORM"] = Token.DrawLineForm,
+            ["DIM"] = Token.Dim,
+            ["CONST"] = Token.Const,
+            ["IF"] = Token.If,
+            ["ENDIF"] = Token.EndIf,
+            ["THEN"] = Token.Then,
+            ["ELSE"] = Token.Else,
+            ["FOR"] = Token.For,
+            ["TO"] = Token.To,
+            ["NEXT"] = Token.Next,
+            ["INPUT"] = Token.Input,
+            ["LET"] = Token.Let,
+            ["GOSUB"] = Token.Gosub,
+            ["RETURN"] = Token.Return,
+            ["END"] = Token.End,
+            ["OR"] = Token.Or,
+            ["AND"] = Token.And,
 
-            TokenMarker = sourceMarker;
+            ["TIMES"] = Token.Times,
+        };
 
-            if (char.IsLetter(lastChar))
+        public IEnumerable<Token> GetTokens()
+        {
+            while (true)
             {
-                Identifer = lastChar.ToString();
-                while (char.IsLetterOrDigit(GetChar()))
-                    Identifer += lastChar;
-                switch (Identifer.ToUpper())
+                GetChar();
+
+                while (lastChar == ' ' || lastChar == '\t' || lastChar == '\r')
+                    GetChar();
+
+                TokenMarker = sourceMarker;
+
+                if (char.IsLetter(lastChar))
                 {
-                    case "PRINT": return Token.Print;
-                    case "PRINTL": return Token.PrintL;
-                    case "DRAWLINE": return Token.DrawLine;
-                    case "DRAWLINEFORM": return Token.DrawLineForm;
-                    case "DIM": return Token.Dim;
-                    case "CONST": return Token.Const;
-                    case "IF": return Token.If;
-                    case "ENDIF": return Token.EndIf;
-                    case "THEN": return Token.Then;
-                    case "ELSE": return Token.Else;
-                    case "FOR": return Token.For;
-                    case "TO": return Token.To;
-                    case "NEXT": return Token.Next;
-                    case "INPUT": return Token.Input;
-                    case "LET": return Token.Let;
-                    case "GOSUB": return Token.Gosub;
-                    case "RETURN": return Token.Return;
-                    case "END": return Token.End;
-                    case "OR": return Token.Or;
-                    case "AND": return Token.And;
-                    case "REM":
-                        while (lastChar != '\n') GetChar();
-                        GetChar();
-                        return GetToken();
-                    default:
-                        return Token.Identifer;
-                }
-            }
+                    Identifer = lastChar.ToString();
+                    while (char.IsLetterOrDigit(GetChar()) || lastChar == '_')
+                        Identifer += lastChar;
 
-            if (char.IsDigit(lastChar))
-            {
-                string num = "";
-                do { num += lastChar; } while (char.IsDigit(GetChar()) || lastChar == '.');
-
-                double real;
-                if (!double.TryParse(num, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out real))
-                    throw new Exception("ERROR while parsing number");
-                Value = new Value(real);
-                return Token.Value;
-            }
+                    if (TokenDictionary.ContainsKey(Identifer))
+                    {
+                        yield return TokenDictionary[Identifer];
+                        continue;
+                    }
 
-            Token tok = Token.Unkown;
-            switch (lastChar)
-            {
-                case '\n': tok = Token.NewLine; break;
-                case '@': tok = Token.Function; break;
-                case '#': tok = Token.Global; break;
-                case ':': tok = Token.Colon; break;
-                case ';': tok = Token.Semicolon; break;
-                case ',': tok = Token.Comma; break;
-                case '=': tok = Token.Equal; break;
-                case '+': tok = Token.Plus; break;
-                case '-': tok = Token.Minus; break;
-                case '/': tok = Token.Slash; break;
-                case '*': tok = Token.Asterisk; break;
-                case '^': tok = Token.Caret; break;
-                case '(': tok = Token.LParen; break;
-                case ')': tok = Token.RParen; break;
-                case '\'':
-                    while (lastChar != '\n') GetChar();
-                    GetChar();
-                    return GetToken();
-                case '<':
-                    GetChar();
-                    if (lastChar == '>') tok = Token.NotEqual;
-                    else if (lastChar == '=') tok = Token.LessEqual;
-                    else return Token.Less;
-                    break;
-                case '>':
-                    GetChar();
-                    if (lastChar == '=') tok = Token.MoreEqual;
-                    else return Token.More;
-                    break;
-                case '"':
-                    string str = "";
-                    while (GetChar() != '"')
+                    switch (Identifer.ToUpper())
                     {
-                        if (lastChar == '\\')
+                        case "DRAWLINEFORM":
+                            foreach (Token t in ReturnAsLine(Token.DrawLineForm))
+                                yield return t;
+                            continue;
+
+                        case "PRINTFORML":
+                            foreach (Token t in ReturnAsLine(Token.PrintFormL))
+                                yield return t;
+                            continue;
+
+                        case "PRINT":
+                            foreach (Token t in ReturnAsLine(Token.Print))
+                                yield return t;
+                            continue;
+
+                        case "REM":
+                            while (lastChar != '\n')
+                                GetChar();
+
+                            continue;
+                        default:
                         {
-                            switch (char.ToLower(GetChar()))
-                            {
-                                case 'n': str += '\n'; break;
-                                case 't': str += '\t'; break;
-                                case '\\': str += '\\'; break;
-                                case '"': str += '"'; break;
-                            }
+                            yield return Token.Identifer;
+
+                            sourceMarker.Pointer--;
+
+                            continue;
                         }
+                    }
+                }
+
+                if (char.IsDigit(lastChar))
+                {
+                    string num = "";
+
+                    do
+                    {
+                        num += lastChar;
+                    }
+                    while (char.IsDigit(GetChar()) || lastChar == '.');
+
+                    if (!double.TryParse(num, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var real))
+                        throw new Exception("ERROR while parsing number");
+
+                    Value = new Value(real);
+                    yield return Token.Value;
+                    
+                    sourceMarker.Pointer--;
+
+                    continue;
+                }
+                
+                switch (lastChar)
+                {
+                    case '\n': yield return Token.NewLine; continue;
+                    case '@': yield return Token.Function; continue;
+                    case '#': yield return Token.Global; continue;
+                    case ':': yield return Token.Colon; continue;
+                    case ';': yield return Token.Semicolon; continue;
+                    case ',': yield return Token.Comma; continue;
+                    case '=': yield return Token.Equal; continue;
+                    case '+': yield return Token.Plus; continue;
+                    case '-': yield return Token.Minus; continue;
+                    case '/': yield return Token.Slash; continue;
+                    case '*': yield return Token.Asterisk; continue;
+                    case '^': yield return Token.Caret; continue;
+                    case '(': yield return Token.LParen; continue;
+                    case ')': yield return Token.RParen; continue;
+                    case '\'':
+                        while (lastChar != '\n')
+                            GetChar();
+                        continue;
+                    case '<':
+                        GetChar();
+                        if (lastChar == '>')
+                            yield return Token.NotEqual;
+                        else if (lastChar == '=')
+                            yield return Token.LessEqual;
+                        else
+                            yield return Token.Less;
+                        continue;
+                    case '>':
+                        GetChar();
+                        if (lastChar == '=')
+                            yield return Token.MoreEqual;
                         else
+                            yield return Token.More;
+                        continue;
+                    case '"':
+                        string str = "";
+                        while (GetChar() != '"')
                         {
-                            str += lastChar;
+                            if (lastChar == '\\')
+                            {
+                                switch (char.ToLower(GetChar()))
+                                {
+                                    case 'n': str += '\n'; break;
+                                    case 't': str += '\t'; break;
+                                    case '\\': str += '\\'; break;
+                                    case '"': str += '"'; break;
+                                }
+                            }
+                            else
+                            {
+                                str += lastChar;
+                            }
                         }
-                    }
-                    Value = new Value(str);
-                    tok = Token.Value;
-                    break;
-                case (char)0:
-                    return Token.EOF;
+                        Value = new Value(str);
+                        yield return Token.Value;
+                        continue;
+                    case (char)0:
+                        yield return Token.EOF;
+                        break;
+                    default:
+                        yield return Token.Unkown;
+                        continue;
+                }
+
+                break;
             }
+        }
+
+        public IEnumerable<Token> ReturnAsLine(Token token)
+        {
+            StringBuilder bodyBuilder = new StringBuilder();
+
+            while (lastChar != '\n')
+                bodyBuilder.Append(GetChar());
+
+            yield return token;
+
+            Value = new Value(bodyBuilder.ToString());
+            yield return Token.Value;
 
-            GetChar();
-            return tok;
+            yield return Token.NewLine;
         }
     }
 }

NTERA.Interpreter/Token.cs

@@ -18,6 +18,8 @@ namespace NTERA.Interpreter
         DrawLineForm,
         Print,
         PrintL,
+        PrintForm,
+        PrintFormL,
 
         //Keywords
         If,
@@ -53,6 +55,7 @@ namespace NTERA.Interpreter
         Or,
         And,
         Not,
+        Times,
 
         LParen,
         RParen,