Make string parsing beter

NTERA.Interpreter/Interpreter.cs

@@ -16,15 +16,15 @@ namespace NTERA.Interpreter
 
         public delegate Value BasicFunction(List<Value> args);
 
-        private readonly IConsole console;
+        protected readonly IConsole Console;
 
-        private Marker lineMarker;
+        protected Marker LineMarker;
 
         private bool exit;
 
         public Interpreter(IConsole console, string input)
         {
-            this.console = console;
+            this.Console = console;
             Lexer = new Lexer(input);
 
             GenerateKeywordDictionary();
@@ -33,7 +33,7 @@ namespace NTERA.Interpreter
 
         private void Error(string text)
         {
-            throw new Exception(text + " at line: " + lineMarker.Line);
+            throw new Exception(text + " at line: " + LineMarker.Line);
         }
 
         protected bool TryAssertToken(Token tok, bool pullNext = true)
@@ -50,7 +50,7 @@ namespace NTERA.Interpreter
                 Error("Expect " + tok + " got " + CurrentToken);
         }
 
-        private IEnumerator<Token> Tokens;
+        protected IEnumerator<Token> Tokens;
 
         public void Exec()
         {
@@ -87,7 +87,7 @@ namespace NTERA.Interpreter
                 return;
             }
 
-            lineMarker = Lexer.TokenMarker;
+            LineMarker = Lexer.TokenMarker;
             Statement();
 
             if (CurrentToken != Token.NewLine && CurrentToken != Token.EOF)
@@ -110,8 +110,6 @@ namespace NTERA.Interpreter
                         break;
                         
                     case Token.Function:
-                    case Token.Dim:
-                    case Token.Const:
                         while (GetNextToken() != Token.NewLine) { }
                         break;
 
@@ -144,8 +142,8 @@ namespace NTERA.Interpreter
                
                 if (!Variables.ContainsKey(Lexer.Identifer)) Variables.Add(Lexer.Identifer, new Value());
 
-                console.WaitInput(new Core.Interop.InputRequest() { InputType = Core.Interop.InputType.StrValue });
-                string input = console.LastInput;
+                Console.WaitInput(new Core.Interop.InputRequest() { InputType = Core.Interop.InputType.StrValue });
+                string input = Console.LastInput;
 
                 if (double.TryParse(input, System.Globalization.NumberStyles.Float, System.Globalization.CultureInfo.InvariantCulture, out var d))
                     Variables[Lexer.Identifer] = new Value(d);
@@ -219,7 +217,19 @@ namespace NTERA.Interpreter
                 {
                     if (Variables.ContainsKey(lexer.Identifer))
                     {
-                        stack.Push(Variables[lexer.Identifer]);
+                        string varName = lexer.Identifer;
+                        int index = 0;
+                        enumerator.MoveNext();
+                        if (enumerator.Current == Token.Colon)
+                        {
+                            enumerator.MoveNext();
+                            index = (int)Expression(lexer, enumerator).Real;
+                        }
+
+                        stack.Push(Variables[varName, index]);
+
+                        i++;
+                        continue;
                     }
                     else if (FunctionDictionary.ContainsKey(lexer.Identifer))
                     {
@@ -271,7 +281,13 @@ namespace NTERA.Interpreter
                 else
                 {
                     if (i == 0)
-                        Error("Empty expression");
+                    {
+                        if (type == LexerType.String)
+                            stack.Push("");
+                        else
+                            Error("Empty expression");
+                    }
+
                     break;
                 }
 

NTERA.Interpreter/Interpreter/Keywords.cs

@@ -30,36 +30,33 @@ namespace NTERA.Interpreter
         [KeywordMethod(Token.Print)]
         private void Print()
         {
-            console.Write(RealExpression().ToString());
+            Console.Write(RealExpression().ToString());
         }
 
         [KeywordMethod(Token.PrintL)]
         private void PrintL()
         {
-            console.PrintSingleLine(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)));
+            Console.PrintSingleLine(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)));
 
             GetNextToken();
         }
-
-
+        
         [KeywordMethod(Token.PrintHtml)]
         private void PrintHtml()
         {
             AssertToken(Token.Value, false);
 
-            console.PrintHtml(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)), true);
+            Console.PrintHtml(ParseFormat(Expression(new Lexer(Lexer.Value, LexerType.String), type: LexerType.String)), true);
 
             GetNextToken();
         }
-
-
+        
         [KeywordMethod(Token.PrintImg)]
         private void PrintImg()
         {
-            console.PrintImg(RealExpression().ToString().Trim().Trim('"'));
+            Console.PrintImg(RealExpression().ToString().Trim().Trim('"'));
         }
-
-
+        
         [KeywordMethod(Token.PrintButton)]
         private void PrintButton()
         {
@@ -70,7 +67,7 @@ namespace NTERA.Interpreter
 
             var value = RealExpression();
 
-            console.PrintButton(text, (long)value.Real);
+            Console.PrintButton(text, (long)value.Real);
         }
 
         private static readonly Regex RealFormatRegex = new Regex("{(.*?)}");
@@ -90,7 +87,7 @@ namespace NTERA.Interpreter
         {
             AssertToken(Token.Value, false);
 
-            console.Write(ParseFormat(Lexer.Value));
+            Console.Write(ParseFormat(Lexer.Value));
 
             GetNextToken();
         }
@@ -100,7 +97,7 @@ namespace NTERA.Interpreter
         {
             AssertToken(Token.Value, false);
 
-            console.PrintSingleLine(ParseFormat(Lexer.Value));
+            Console.PrintSingleLine(ParseFormat(Lexer.Value));
 
             GetNextToken();
         }
@@ -108,19 +105,19 @@ namespace NTERA.Interpreter
         [KeywordMethod(Token.DrawLine)]
         private void DrawLine()
         {
-            console.PrintBar();
+            Console.PrintBar();
         }
 
         [KeywordMethod(Token.DrawLineForm)]
         private void DrawLineForm()
         {
-            console.printCustomBar(RealExpression().ToString().Trim());
+            Console.printCustomBar(RealExpression().ToString().Trim());
         }
 
         [KeywordMethod(Token.CustomDrawLine)]
         private void CustomDrawLine()
         {
-            console.printCustomBar(RealExpression().ToString().Trim());
+            Console.printCustomBar(RealExpression().ToString().Trim());
         }
 
         [KeywordMethod(Token.Alignment)]
@@ -128,7 +125,7 @@ namespace NTERA.Interpreter
         {
             AssertToken(Token.Value, false);
 
-            console.Alignment = (DisplayLineAlignment)Enum.Parse(typeof(DisplayLineAlignment), Lexer.Value);
+            Console.Alignment = (DisplayLineAlignment)Enum.Parse(typeof(DisplayLineAlignment), Lexer.Value);
 
             GetNextToken();
         }
@@ -141,7 +138,7 @@ namespace NTERA.Interpreter
                 int argb = (int)((int)RealExpression().Real | 0xFF000000);
                 Color c = Color.FromArgb(argb);
 
-                console.SetStringStyle(c);
+                Console.SetStringStyle(c);
                 return;
             }
 
@@ -160,7 +157,7 @@ namespace NTERA.Interpreter
 
                 int b = (int)Lexer.Value;
 
-                console.SetStringStyle(Color.FromArgb(r, g, b));
+                Console.SetStringStyle(Color.FromArgb(r, g, b));
 
                 GetNextToken();
             }
@@ -169,14 +166,14 @@ namespace NTERA.Interpreter
                 int argb = (int)((int)Lexer.Value.Real | 0xFF000000);
                 Color c = Color.FromArgb(argb);
 
-                console.SetStringStyle(c);
+                Console.SetStringStyle(c);
             }
         }
 
         [KeywordMethod(Token.ResetColor)]
         private void ResetColor()
         {
-            console.ResetStyle();
+            Console.ResetStyle();
 
             GetNextToken();
         }
@@ -252,7 +249,7 @@ namespace NTERA.Interpreter
                 value = RealExpression();
             }
             else
-                value = StringExpression();
+                value = ParseFormat(StringExpression());
 
 
             if (appending)
@@ -274,12 +271,12 @@ namespace NTERA.Interpreter
 
             if (Loops.ContainsKey(var))
             {
-                Loops[var] = lineMarker;
+                Loops[var] = LineMarker;
             }
             else
             {
                 Variables[var] = v;
-                Loops.Add(var, lineMarker);
+                Loops.Add(var, LineMarker);
             }
 
             AssertToken(Token.To, false);
@@ -291,7 +288,7 @@ namespace NTERA.Interpreter
             {
                 while (true)
                 {
-                    while (!(GetNextToken() == Token.Identifer && prevToken == Token.Next)) { }
+                    while (!(GetNextToken() == Token.Identifer && previousToken == Token.Next)) { }
 
                     if (Lexer.Identifer == var)
                     {

NTERA.Interpreter/Lexer.cs

@@ -107,107 +107,122 @@ namespace NTERA.Interpreter
             return c == '%' || c == '{';
         }
 
-        public IEnumerable<Token> GetTokens()
+        private Token DetermineToken(char c)
         {
-            while (true)
+            if (TokenCharDictionary.TryGetValue(currentChar, out Token charToken))
+                return charToken;
+
+            switch (currentChar)
             {
-                while (IsWhitespace(GetNextChar()) && Type != LexerType.String) { }
+                case ';': //semicolon is comment
+                    while (currentChar != '\n')
+                        GetNextChar();
+
+                    return Token.NewLine;
+                case '<':
+                    if (!Type.HasFlag(LexerType.Real))
+                        break;
+
+                    if (GetNextChar(true) == '>')
+                    {
+                        GetNextChar();
+                        return Token.NotEqual;
+                    }
+                    else if (GetNextChar(true) == '=')
+                    {
+                        GetNextChar();
+                        return Token.LessEqual;
+                    }
+                    else
+                        return Token.Less;
 
-                TokenMarker = sourceMarker;
+                case '>':
+                    if (!Type.HasFlag(LexerType.Real))
+                        break;
 
-                if (TokenCharDictionary.TryGetValue(currentChar, out Token charToken))
-                {
-                    yield return charToken;
-                    continue;
-                }
+                    if (GetNextChar(true) == '=')
+                    {
+                        GetNextChar();
+                        return Token.MoreEqual;
+                    }
+                    else
+                        return Token.More;
+
+                case '+':
+                    if (GetNextChar(true) == '=')
+                    {
+                        GetNextChar();
+                        return Token.Append;
+                    }
+                    else
+                        return Token.Plus;
 
-                switch (currentChar)
-                {
-                    case ';': //semicolon is comment
-                        while (currentChar != '\n')
-                            GetNextChar();
-                        continue;
-                    case '<':
-                        if (!Type.HasFlag(LexerType.Real))
-                            break;
-                        
-                        if (GetNextChar(true) == '>')
-                        {
-                            GetNextChar();
-                            yield return Token.NotEqual;
-                        }
-                        else if (GetNextChar(true) == '=')
-                        {
-                            GetNextChar();
-                            yield return Token.LessEqual;
-                        }
-                        else
-                            yield return Token.Less;
-                        continue;
-                    case '>':
-                        if (!Type.HasFlag(LexerType.Real))
-                            break;
+                case '%':
+
+                    StringBuilder builder = new StringBuilder();
+                    while (GetNextChar() != '%')
+                        builder.Append(currentChar);
+
+                    Value = $"%{builder}%";
+                    return Token.Value;
 
-                        if (GetNextChar(true) == '=')
+                case '"':
+                    string str = "";
+                    while (GetNextChar() != '"')
+                    {
+                        if (currentChar == '\\')
                         {
-                            GetNextChar();
-                            yield return Token.MoreEqual;
+                            switch (char.ToLower(GetNextChar()))
+                            {
+                                case 'n': str += '\n'; break;
+                                case 't': str += '\t'; break;
+                                case '\\': str += '\\'; break;
+                                case '"': str += '"'; break;
+                            }
                         }
                         else
-                            yield return Token.More;
-                        continue;
-                    case '+':
-                        if (GetNextChar(true) == '=')
                         {
-                            GetNextChar();
-                            yield return Token.Append;
+                            str += currentChar;
                         }
-                        else
-                            yield return Token.Plus;
-                        continue;
-                    case '%':
+                    }
+                    Value = new Value(str);
+                    return Token.Value;
+
+                case (char)0:
+                    return Token.EOF;
+            }
 
-                        StringBuilder builder = new StringBuilder();
-                        while (GetNextChar() != '%')
-                            builder.Append(currentChar);
+            return Token.Unknown;
+        }
 
-                        Value = $"%{builder}%";
-                        yield return Token.Value;
+        public IEnumerable<Token> GetTokens()
+        {
+            while (true)
+            {
+                while (IsWhitespace(GetNextChar()) && Type != LexerType.String || currentChar == '\r') { }
 
-                        continue;
-                    case '"':
-                        string str = "";
-                        while (GetNextChar() != '"')
-                        {
-                            if (currentChar == '\\')
-                            {
-                                switch (char.ToLower(GetNextChar()))
-                                {
-                                    case 'n': str += '\n'; break;
-                                    case 't': str += '\t'; break;
-                                    case '\\': str += '\\'; break;
-                                    case '"': str += '"'; break;
-                                }
-                            }
-                            else
-                            {
-                                str += currentChar;
-                            }
-                        }
-                        Value = new Value(str);
-                        yield return Token.Value;
-                        continue;
-                    case (char)0:
-                        yield return Token.EOF;
-                        yield break;
+                TokenMarker = sourceMarker;
+
+                Token token = DetermineToken(currentChar);
+
+                if (token == Token.EOF)
+                {
+                    yield return Token.EOF;
+                    yield break;
+                }
+
+                if (token != Token.Unknown)
+                {
+                    yield return token;
+                    continue;
                 }
 
                 StringBuilder bodyBuilder = new StringBuilder(currentChar.ToString());
 
-                while (!TokenCharDictionary.ContainsKey(GetNextChar(true)) 
-                       && !IsEndOfLine(GetNextChar(true)) 
-                       && (!IsWhitespace(GetNextChar(true)) || Type == LexerType.String)
-                       && (!IsEscape(GetNextChar(true)) || Type != LexerType.String))
+                while ((!IsEscape(GetNextChar(true)) || Type != LexerType.String)
+                        && DetermineToken(GetNextChar(true)) == Token.Unknown
+                        && (!IsWhitespace(GetNextChar(true)) || Type == LexerType.String)
+                        && GetNextChar(true) != '\r')
                 {
                     bodyBuilder.Append(GetNextChar());
                 }
@@ -230,12 +245,15 @@ namespace NTERA.Interpreter
 
                 Identifer = bodyBuilder.ToString();
 
-                if (TokenDictionary.TryGetValue(Identifer, out Token token))
+                if (TokenDictionary.TryGetValue(Identifer, out token))
                 {
                     yield return token;
                     continue;
                 }
 
+                if (Type == LexerType.String && char.IsWhiteSpace(Identifer[0]))
+                    Identifer = Identifer.Substring(1);
+
                 if (TokenLineDictionary.TryGetValue(Identifer, out token))
                 {
                     bodyBuilder = new StringBuilder();
@@ -245,7 +263,12 @@ namespace NTERA.Interpreter
                     
                     yield return token;
 
-                    Value = new Value(bodyBuilder.ToString().Substring(1));
+                    string strValue = bodyBuilder.ToString();
+
+                    if (strValue.Length > 0 && char.IsWhiteSpace(strValue[0]))
+                        strValue = strValue.Substring(1);
+
+                    Value = new Value(strValue);
                     yield return Token.Value;
 
                     yield return currentChar == '\0' ? Token.EOF : Token.NewLine;