better whitespace handling

README.md

@@ -60,6 +60,7 @@ EnableBatching=True              ;Indicates whether batching of translations sho
 TrimAllText=True                 ;Indicates whether spaces in front and behind translation candidates should be removed before translation
 UseStaticTranslations=True       ;Indicates whether or not to use translations from the included static translation cache
 OverrideFont=                    ;Overrides the fonts used for texts when updating text components. NOTE: Only works for UGUI
+WhitespaceHandlingStrategy=TrimPerNewline ;Indicates how whitespace/newline removal should be handled before attempting translation. Can be ["TrimPerNewline", "AllOccurrences"]
 
 [Http]
 UserAgent=                       ;Override the user agent used by APIs requiring a user agent

src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs

@@ -120,7 +120,19 @@ namespace XUnity.AutoTranslator.Plugin.Core
             Logger.Current = new ConsoleLogger();
          }
 
-         Settings.Configure();
+         try
+         {
+            Settings.Configure();
+         }
+         catch( Exception e )
+         {
+            Logger.Current.Error( e, "An error occurred during configuration. Shutting plugin down." );
+
+            _endpoint = null;
+            Settings.IsShutdown = true;
+
+            return;
+         }
 
          if( Settings.EnableConsole ) DebugConsole.Enable();
 

src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs

@@ -4,6 +4,7 @@ using System.IO;
 using System.Linq;
 using System.Text;
 using XUnity.AutoTranslator.Plugin.Core.Constants;
+using XUnity.AutoTranslator.Plugin.Core.Debugging;
 
 namespace XUnity.AutoTranslator.Plugin.Core.Configuration
 {
@@ -70,6 +71,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
       public static bool UseStaticTranslations;
       public static string OverrideFont;
       public static string UserAgent;
+      public static WhitespaceHandlingStrategy WhitespaceHandlingStrategy;
 
       public static bool CopyToClipboard;
       public static int MaxClipboardCopyCharacters;
@@ -111,7 +113,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
          MaxCharactersPerTranslation = Config.Current.Preferences[ "Behaviour" ][ "MaxCharactersPerTranslation" ].GetOrDefault( 200 );
          IgnoreWhitespaceInDialogue = Config.Current.Preferences[ "Behaviour" ][ "IgnoreWhitespaceInDialogue" ].GetOrDefault( Types.AdvEngine == null );
          IgnoreWhitespaceInNGUI = Config.Current.Preferences[ "Behaviour" ][ "IgnoreWhitespaceInNGUI" ].GetOrDefault( true );
-         MinDialogueChars = Config.Current.Preferences[ "Behaviour" ][ "MinDialogueChars" ].GetOrDefault( 20 );
+         MinDialogueChars = Config.Current.Preferences[ "Behaviour" ][ "MinDialogueChars" ].GetOrDefault( 18 );
          ForceSplitTextAfterCharacters = Config.Current.Preferences[ "Behaviour" ][ "ForceSplitTextAfterCharacters" ].GetOrDefault( 0 );
          CopyToClipboard = Config.Current.Preferences[ "Behaviour" ][ "CopyToClipboard" ].GetOrDefault( false );
          MaxClipboardCopyCharacters = Config.Current.Preferences[ "Behaviour" ][ "MaxClipboardCopyCharacters" ].GetOrDefault( 450 );
@@ -121,6 +123,18 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
          UseStaticTranslations = Config.Current.Preferences[ "Behaviour" ][ "UseStaticTranslations" ].GetOrDefault( true );
          OverrideFont = Config.Current.Preferences[ "Behaviour" ][ "OverrideFont" ].GetOrDefault( string.Empty );
 
+         // special handling because of P/Invoke / exe name handling
+         try
+         {
+            WhitespaceHandlingStrategy = Config.Current.Preferences[ "Behaviour" ][ "WhitespaceHandlingStrategy" ].GetOrDefault( GetDefaultWhitespaceHandlingStrategy() );
+         }
+         catch( Exception e )
+         {
+            WhitespaceHandlingStrategy = WhitespaceHandlingStrategy.TrimPerNewline;
+
+            Logger.Current.Warn( e, "An error occurred while configuring 'WhitespaceHandlingStrategy'. Using default." );
+         }
+
          UserAgent = Config.Current.Preferences[ "Http" ][ "UserAgent" ].GetOrDefault( string.Empty );
 
          GoogleAPIKey = Config.Current.Preferences[ "GoogleLegitimate" ][ "GoogleAPIKey" ].GetOrDefault( "" );
@@ -177,5 +191,17 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
          }
          return defaultUserAgent;
       }
+
+      public static WhitespaceHandlingStrategy GetDefaultWhitespaceHandlingStrategy()
+      {
+         string exe = Path.GetFileNameWithoutExtension( Kernel32.ApplicationPath );
+         if( exe.StartsWith( "charastudio", StringComparison.OrdinalIgnoreCase )
+            || exe.StartsWith( "koikatu", StringComparison.OrdinalIgnoreCase ) )
+         {
+            // for legacy reasons, lets not change all the keys of existing translations
+            return WhitespaceHandlingStrategy.AllOccurrences;
+         }
+         return WhitespaceHandlingStrategy.TrimPerNewline;
+      }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Debugging/Kernel32.cs

@@ -8,6 +8,9 @@ namespace XUnity.AutoTranslator.Plugin.Core.Debugging
 {
    public static class Kernel32
    {
+      [DllImport( "kernel32.dll", CharSet = CharSet.Unicode )]
+      public static extern int GetModuleFileNameW( HandleRef hModule, StringBuilder buffer, int length );
+
       [DllImport( "kernel32.dll", SetLastError = true )]
       public static extern bool AllocConsole();
 
@@ -32,5 +35,15 @@ namespace XUnity.AutoTranslator.Plugin.Core.Debugging
 
       [DllImport( "kernel32.dll", ExactSpelling = true, SetLastError = true )]
       public static extern bool CloseHandle( IntPtr handle );
+
+      public static string ApplicationPath
+      {
+         get
+         {
+            StringBuilder stringBuilder = new StringBuilder( 260 );
+            GetModuleFileNameW( new HandleRef( null, IntPtr.Zero ), stringBuilder, stringBuilder.Capacity );
+            return stringBuilder.ToString();
+         }
+      }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Extensions/StringExtensions.cs

@@ -58,6 +58,9 @@ namespace XUnity.AutoTranslator.Plugin.Core.Extensions
          '.'
       };
 
+      private static readonly char[] NewlinesCharacters = new char[] { '\r', '\n' };
+      private static readonly char[] WhitespacesAndNewlines = new char[] { '\r', '\n', ' ', '　' };
+
       public static TemplatedString TemplatizeByNumbers( this string str )
       {
          var dict = new Dictionary<string, string>();
@@ -193,10 +196,41 @@ namespace XUnity.AutoTranslator.Plugin.Core.Extensions
          return text;
       }
 
-      public static string RemoveWhitespace( this string text )
+      public static string RemoveWhitespaceAndNewlines( this string text )
       {
-         // Japanese whitespace, wtf
-         return text.Replace( "\n", "" ).Replace( "\r", "" ).Replace( " ", "" ).Replace( "　", "" );
+         var builder = new StringBuilder( text.Length );
+         if( Settings.WhitespaceHandlingStrategy == WhitespaceHandlingStrategy.AllOccurrences )
+         {
+            for( int i = 0 ; i < text.Length ; i++ )
+            {
+               var c = text[ i ];
+               switch( c )
+               {
+                  case '\n':
+                  case '\r':
+                  case ' ':
+                  case '　':
+                     break;
+                  default:
+                     builder.Append( c );
+                     break;
+               }
+            }
+         }
+         else // if( Settings.WhitespaceHandlingStrategy == WhitespaceHandlingStrategy.TrimPerNewline )
+         {
+            var lines = text.Split( NewlinesCharacters, StringSplitOptions.RemoveEmptyEntries );
+            for( int i = 0 ; i < lines.Length ; i++ )
+            {
+               var line = lines[ i ].Trim( WhitespacesAndNewlines );
+               for( int j = 0 ; j < line.Length ; j++ )
+               {
+                  var c = line[ j ];
+                  builder.Append( c );
+               }
+            }
+         }
+         return builder.ToString();
       }
 
       public static bool ContainsNumbers( this string text )

src/XUnity.AutoTranslator.Plugin.Core/TranslationKey.cs

@@ -14,9 +14,9 @@ namespace XUnity.AutoTranslator.Plugin.Core
          OriginalText = key;
 
          if( !neverRemoveWhitespace
-            && ( ( Settings.IgnoreWhitespaceInDialogue && key.Length > Settings.MinDialogueChars ) || ( ui.IgnoreAllWhitespace() ) ) )
+            && ( ( Settings.IgnoreWhitespaceInDialogue && key.Length > Settings.MinDialogueChars ) || ( Settings.IgnoreWhitespaceInNGUI && ui.IgnoreAllWhitespace() ) ) )
          {
-            RelevantText = key.RemoveWhitespace();
+            RelevantText = key.RemoveWhitespaceAndNewlines();
          }
          else
          {

src/XUnity.AutoTranslator.Plugin.Core/WhitespaceHandlingStrategy.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace XUnity.AutoTranslator.Plugin.Core
+{
+   public enum WhitespaceHandlingStrategy
+   {
+      AllOccurrences,
+      TrimPerNewline
+   }
+}