romaji post processing

bing translate batching
spelling errors
README update

README.md

@@ -141,6 +141,7 @@ ForceUIResizing=True             ;Indicates whether the UI resize behavior shoul
 IgnoreTextStartingWith=\u180e;   ;Indicates that the plugin should ignore any strings starting with certain characters. This is a list seperated by ';'.
 TextGetterCompatibilityMode=False ;Indicates whether or not to enable "Text Getter Compatibility Mode". Should only be enabled if required by the game. 
 GameLogTextPaths=                ;Indicates specific paths for game objects that the game uses as "log components", where it continuously appends or prepends text to. Requires expert knowledge to setup. This is a list seperated by ';'.
+RomajiPostProcessing=RemoveAllDiacritics;RemoveApostrophes ;Indicates what type of post processing to do on 'translated' romaji texts. This can be important in certain games because the font used does not support various diacritics properly. This is a list seperated by ';'. Possible values: ["RemoveAllDiacritics", "ReplaceMacronWithCircumflex", "RemoveApostrophes"]
 
 [Texture]
 TextureDirectory=Translation\Texture ;Directory to dump textures to, and root of directories to load images from. Can use placeholder: {GameExeName}
@@ -229,6 +230,14 @@ The following aims at reducing the number of requests send to the translation en
  * `UseStaticTranslations`: Enables usage of internal lookup dictionary of various english-to-japanese terms.
  * `MaxCharactersPerTranslation`: Specifies the maximum length of a text to translate. Any texts longer than this is ignored by the plugin. Cannot be greater than 500.
 
+#### Romaji 'translation'
+One of the possible values as output `Language` is 'romaji'. If you choose this as language, you will find that games often has problems showing the translations because the font does not understand the special characters used, for example the [macron diacritic](https://en.wikipedia.org/wiki/Macron_(diacritic)).
+
+To rememdy this, post processing can be applied to translations when 'romaji' is chosen as `Language`. This is done through the option `RomajiPostProcessing`. This option is a ';'-seperated list of values:
+ * `RemoveAllDiacritics`: Remove all diacritics from the translated text
+ * `ReplaceMacronWithCircumflex`: Replaces the macron diacritic with a circumflex.
+ * `RemoveApostrophes`: Some translators might decide to include apostrophes after the 'n'-character. Applying this option removes those.
+
 #### Other Options
  * `TextGetterCompatibilityMode`: This mode fools the game into thinking that the text displayed is not translated. This is required if the game uses text displayed to the user to determine what logic to execute. You can easily determine if this is required if you can see the functionality works fine if you toggle the translation off (hotkey: ALT+T).
  * `IgnoreTextStartingWith`: Disable translation for any texts starting with values in this ';-separated' setting. The [default value](https://www.charbase.com/180e-unicode-mongolian-vowel-separator) is an invisible character that takes up no space.
@@ -584,7 +593,7 @@ internal class YandexTranslateEndpoint : HttpEndpoint
    public override void Initialize( IInitializationContext context )
    {
       _key = context.GetOrCreateSetting( "Yandex", "YandexAPIKey", "" );
-      context.DisableCerfificateChecksFor( "translate.yandex.net" );
+      context.DisableCertificateChecksFor( "translate.yandex.net" );
 
       // if the plugin cannot be enabled, simply throw so the user cannot select the plugin
       if( string.IsNullOrEmpty( _key ) ) throw new Exception( "The YandexTranslate endpoint requires an API key which has not been provided." );
@@ -627,7 +636,7 @@ internal class YandexTranslateEndpoint : HttpEndpoint
 ```
 
 This plugin extends from `HttpEndpoint`. Let's look at the three methods it overrides:
- * `Initialize` is used to read the API key the user has configured. In addition it calls `context.DisableCerfificateChecksFor( "translate.yandex.net" )` in order to disable the certificate check for this specific hostname. If this is neglected, SSL will fail in most versions of Unity. Finally, it throws an exception if the plugin cannot be used with the specified configuration.
+ * `Initialize` is used to read the API key the user has configured. In addition it calls `context.DisableCertificateChecksFor( "translate.yandex.net" )` in order to disable the certificate check for this specific hostname. If this is neglected, SSL will fail in most versions of Unity. Finally, it throws an exception if the plugin cannot be used with the specified configuration.
  * `OnCreateRequest` is used to construct the `XUnityWebRequest` object that will be sent to the external endpoint. The call to `context.Complete( request )` specifies the request to use.
  * `OnExtractTranslation` is used to extract the text from the response returned from the web server.
 

src/Translators/BaiduTranslate/BaiduTranslateEndpoint.cs

@@ -32,7 +32,7 @@ namespace BaiduTranslate
          if( string.IsNullOrEmpty( _appId ) ) throw new ArgumentException( "The BaiduTranslate endpoint requires an App Id which has not been provided." );
          if( string.IsNullOrEmpty( _appSecret ) ) throw new ArgumentException( "The BaiduTranslate endpoint requires an App Secret which has not been provided." );
 
-         context.DisableCerfificateChecksFor( "api.fanyi.baidu.com" );
+         context.DisableCertificateChecksFor( "api.fanyi.baidu.com" );
 
          // frankly, I have no idea what languages this does, or does not support...
       }

src/Translators/BingTranslate/BingTranslateEndpoint.cs

@@ -3,6 +3,7 @@ using System.Collections;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
+using System.Linq;
 using System.Net;
 using System.Reflection;
 using System.Text;
@@ -30,6 +31,8 @@ namespace BingTranslate
       private static readonly string HttpsTranslateUserSite = "https://www.bing.com/translator";
       private static readonly string RequestTemplate = "&text={0}&from={1}&to={2}";
       private static readonly Random RandomNumbers = new Random();
+      private static readonly string TranslationSeparator = "---";
+      private static readonly string TranslationSeparatorWithNewlines = "\r\n---\r\n";
 
       private static readonly string[] Accepts = new string[] { "*/*" };
       private static readonly string[] AcceptLanguages = new string[] { null, "en-US,en;q=0.9", "en-US", "en" };
@@ -61,10 +64,12 @@ namespace BingTranslate
 
       public override string FriendlyName => "Bing Translator";
 
+      public override int MaxTranslationsPerRequest => 10;
+
       public override void Initialize( IInitializationContext context )
       {
          // Configure service points / service point manager
-         context.DisableCerfificateChecksFor( "www.bing.com" );
+         context.DisableCertificateChecksFor( "www.bing.com" );
 
          if( !SupportedLanguages.Contains( context.SourceLanguage ) ) throw new Exception( $"The source language '{context.SourceLanguage}' is not supported." );
          if( !SupportedLanguages.Contains( context.DestinationLanguage ) ) throw new Exception( $"The destination language '{context.DestinationLanguage}' is not supported." );
@@ -72,17 +77,14 @@ namespace BingTranslate
 
       public override IEnumerator OnBeforeTranslate( IHttpTranslationContext context )
       {
-         if( !_hasSetup || AutoTranslatorState.TranslationCount % _resetAfter == 0 )
+         if( !_hasSetup || _translationCount % _resetAfter == 0 )
          {
             _resetAfter = RandomNumbers.Next( 75, 125 );
             _hasSetup = true;
 
             // Setup TKK and cookies
             var enumerator = SetupIGAndIID();
-            while( enumerator.MoveNext() )
-            {
-               yield return enumerator.Current;
-            }
+            while( enumerator.MoveNext() ) yield return enumerator.Current;
          }
       }
 
@@ -100,9 +102,11 @@ namespace BingTranslate
             address = string.Format( HttpsServicePointTemplateUrl, _ig, _iid, _translationCount );
          }
 
+         var requestData = string.Join( TranslationSeparatorWithNewlines, context.UntranslatedTexts );
+
          var data = string.Format(
             RequestTemplate,
-            Uri.EscapeDataString( context.UntranslatedText ),
+            Uri.EscapeDataString( requestData ),
             context.SourceLanguage,
             context.DestinationLanguage );
 
@@ -124,17 +128,40 @@ namespace BingTranslate
          var obj = JSON.Parse( context.Response.Data );
 
          var code = obj[ "statusCode" ].AsInt;
-         if( code != 200 )
-         {
-            return;
-         }
+         if( code != 200 ) context.Fail( "Bad response code received from service: " + code );
 
          var token = obj[ "translationResponse" ].ToString();
-         token = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
+         var allTranslatedText = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
+
+         if( context.UntranslatedTexts.Length > 1 )
+         {
+            var translatedTexts = allTranslatedText
+               .Split( new[] { TranslationSeparator }, StringSplitOptions.RemoveEmptyEntries )
+               .Select( x => x.Trim( ' ', '\r', '\n' ) )
+               .ToArray();
 
-         var translated = token;
+            if( translatedTexts.Length != context.UntranslatedTexts.Length ) context.Fail( "Batch operation received incorrect number of translations." );
 
-         context.Complete( translated );
+            // lets fix the translation line breaks
+            for( int i = 0 ; i < translatedTexts.Length ; i++ )
+            {
+               var translatedText = translatedTexts[ i ];
+               var untranslatedText = context.UntranslatedTexts[ i ];
+
+               if( !untranslatedText.Contains( "\n" ) )
+               {
+                  translatedText = translatedText.Replace( "\r\n", " " ).Replace( "\n", " " ).Replace( "  ", " " );
+               }
+
+               translatedTexts[ i ] = translatedText;
+            }
+
+            context.Complete( translatedTexts );
+         }
+         else
+         {
+            context.Complete( allTranslatedText );
+         }
       }
 
       private XUnityWebRequest CreateWebSiteRequest()

src/Translators/BingTranslateLegitimate/BingTranslateLegitimateEndpoint.cs

@@ -48,7 +48,7 @@ namespace BingTranslateLegitimate
          if( string.IsNullOrEmpty( _key ) ) throw new Exception( "The BingTranslateLegitimate endpoint requires an API key which has not been provided." );
 
          // Configure service points / service point manager
-         context.DisableCerfificateChecksFor( "api.cognitive.microsofttranslator.com" );
+         context.DisableCertificateChecksFor( "api.cognitive.microsofttranslator.com" );
 
          if( !SupportedLanguages.Contains( context.SourceLanguage ) ) throw new Exception( $"The source language '{context.SourceLanguage}' is not supported." );
          if( !SupportedLanguages.Contains( context.DestinationLanguage ) ) throw new Exception( $"The destination language '{context.DestinationLanguage}' is not supported." );

src/Translators/CustomTranslate/CustomTranslateEndpoint.cs

@@ -31,7 +31,7 @@ namespace CustomTranslate
          if( string.IsNullOrEmpty( _endpoint ) ) throw new ArgumentException( "The custom endpoint requires a url which has not been provided." );
 
          var uri = new Uri( _endpoint );
-         context.DisableCerfificateChecksFor( uri.Host );
+         context.DisableCertificateChecksFor( uri.Host );
 
          _friendlyName += " (" + uri.Host + ")";
       }

src/Translators/GoogleTranslate/GoogleTranslateEndpoint.cs

@@ -59,7 +59,7 @@ namespace GoogleTranslate
 
       public override void Initialize( IInitializationContext context )
       {
-         context.DisableCerfificateChecksFor( "translate.google.com", "translate.googleapis.com" );
+         context.DisableCertificateChecksFor( "translate.google.com", "translate.googleapis.com" );
 
          if( context.DestinationLanguage == "romaji" )
          {
@@ -148,7 +148,6 @@ namespace GoogleTranslate
          }
 
          var allTranslation = lineBuilder.ToString();
-
          if( context.UntranslatedTexts.Length == 1 )
          {
             context.Complete( allTranslation );

src/Translators/GoogleTranslateLegitimate/GoogleTranslateLegitimateEndpoint.cs

@@ -36,7 +36,7 @@ namespace GoogleTranslateLegitimate
          if( string.IsNullOrEmpty( _key ) ) throw new Exception( "The GoogleTranslateLegitimate endpoint requires an API key which has not been provided." );
 
          // Configure service points / service point manager
-         context.DisableCerfificateChecksFor( "translation.googleapis.com" );
+         context.DisableCertificateChecksFor( "translation.googleapis.com" );
 
          if( !SupportedLanguages.Contains( context.SourceLanguage ) ) throw new Exception( $"The source language '{context.SourceLanguage}' is not supported." );
          if( !SupportedLanguages.Contains( context.DestinationLanguage ) ) throw new Exception( $"The destination language '{context.DestinationLanguage}' is not supported." );

src/Translators/YandexTranslate/YandexTranslateEndpoint.cs

@@ -29,7 +29,7 @@ namespace YandexTranslate
       public override void Initialize( IInitializationContext context )
       {
          _key = context.GetOrCreateSetting( "Yandex", "YandexAPIKey", "" );
-         context.DisableCerfificateChecksFor( "translate.yandex.net" );
+         context.DisableCertificateChecksFor( "translate.yandex.net" );
 
          // if the plugin cannot be enabled, simply throw so the user cannot select the plugin
          if( string.IsNullOrEmpty( _key ) ) throw new Exception( "The YandexTranslate endpoint requires an API key which has not been provided." );

src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs

@@ -1943,11 +1943,11 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
       private IEnumerator EnableBatchingAfterDelay()
       {
-         yield return new WaitForSeconds( 60 );
+         yield return new WaitForSeconds( 240 );
 
          _batchLogicHasFailed = false;
 
-         XuaLogger.Current.Error( "Re-enabled batching." );
+         XuaLogger.Current.Info( "Re-enabled batching." );
       }
 
       void Awake()
@@ -2212,17 +2212,25 @@ namespace XUnity.AutoTranslator.Plugin.Core
                var translatedText = translatedTexts[ i ];
                if( !string.IsNullOrEmpty( translatedText ) )
                {
+                  translatedText = job.Key.RepairTemplate( translatedText );
+
+                  if( Settings.RomajiPostProcessing != RomajiPostProcessing.None && Settings.Language == Settings.Romaji )
+                  {
+                     translatedText = RomanizationHelper.PostProcess( translatedText, Settings.RomajiPostProcessing );
+                  }
+
                   if( Settings.ForceSplitTextAfterCharacters > 0 )
                   {
                      translatedText = translatedText.SplitToLines( Settings.ForceSplitTextAfterCharacters, '\n', ' ', '　' );
                   }
-                  job.TranslatedText = job.Key.RepairTemplate( translatedText );
+
+                  job.TranslatedText = translatedText;
 
                   QueueNewTranslationForDisk( job.Key, translatedText );
                   _completedJobs.Add( job );
                }
 
-               AddTranslation( job.Key, job.TranslatedText );
+               AddTranslation( job.Key, translatedText );
                job.State = TranslationJobState.Succeeded;
                _ongoingJobs.Remove( job.Key.GetDictionaryLookupKey() );
 
@@ -2273,26 +2281,35 @@ namespace XUnity.AutoTranslator.Plugin.Core
          var translatedText = translatedTexts[ 0 ];
 
          Settings.TranslationCount++;
-         XuaLogger.Current.Info( $"Completed: '{job.Key.GetDictionaryLookupKey()}' => '{translatedText}'" );
 
          _consecutiveErrors = 0;
 
          if( !string.IsNullOrEmpty( translatedText ) )
          {
+            translatedText = job.Key.RepairTemplate( translatedText );
+
+            if( Settings.RomajiPostProcessing != RomajiPostProcessing.None && Settings.Language == Settings.Romaji )
+            {
+               translatedText = RomanizationHelper.PostProcess( translatedText, Settings.RomajiPostProcessing );
+            }
+            
             if( Settings.ForceSplitTextAfterCharacters > 0 )
             {
                translatedText = translatedText.SplitToLines( Settings.ForceSplitTextAfterCharacters, '\n', ' ', '　' );
             }
-            job.TranslatedText = job.Key.RepairTemplate( translatedText );
+
+            job.TranslatedText = translatedText;
 
             QueueNewTranslationForDisk( job.Key, translatedText );
             _completedJobs.Add( job );
          }
 
-         AddTranslation( job.Key, job.TranslatedText );
+         AddTranslation( job.Key, translatedText );
          job.State = TranslationJobState.Succeeded;
          _ongoingJobs.Remove( job.Key.GetDictionaryLookupKey() );
 
+         XuaLogger.Current.Info( $"Completed: '{job.Key.GetDictionaryLookupKey()}' => '{translatedText}'" );
+
          if( !Settings.IsShutdown )
          {
             if( Settings.TranslationCount > Settings.MaxTranslationsBeforeShutdown )

src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs

@@ -20,6 +20,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
       public static readonly string DefaultLanguage = "en";
       public static readonly string DefaultFromLanguage = "ja";
       public static readonly string EnglishLanguage = "en";
+      public static readonly string Romaji = "romaji";
       public static readonly int MaxErrors = 5;
       public static readonly float ClipboardDebounceTime = 1f;
       public static readonly int MaxTranslationsBeforeShutdown = 8000;
@@ -78,6 +79,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
       public static string[] IgnoreTextStartingWith;
       public static HashSet<string> GameLogTextPaths;
       public static bool TextGetterCompatibilityMode;
+      public static RomajiPostProcessing RomajiPostProcessing;
 
       public static string TextureDirectory;
       public static bool EnableTextureTranslation;
@@ -113,8 +115,8 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
 
          ServiceEndpoint = PluginEnvironment.Current.Preferences.GetOrDefault( "Service", "Endpoint", KnownTranslateEndpointNames.GoogleTranslate );
 
-         Language = PluginEnvironment.Current.Preferences.GetOrDefault( "General", "Language", DefaultLanguage );
-         FromLanguage = PluginEnvironment.Current.Preferences.GetOrDefault( "General", "FromLanguage", DefaultFromLanguage );
+         Language = string.Intern( PluginEnvironment.Current.Preferences.GetOrDefault( "General", "Language", DefaultLanguage ) );
+         FromLanguage = string.Intern( PluginEnvironment.Current.Preferences.GetOrDefault( "General", "FromLanguage", DefaultFromLanguage ) );
 
          TranslationDirectory = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "Directory", "Translation" );
          OutputFile = PluginEnvironment.Current.Preferences.GetOrDefault( "Files", "OutputFile", @"Translation\_AutoGeneratedTranslations.{lang}.txt" );
@@ -148,6 +150,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
             ?.Split( new[] { ';' }, StringSplitOptions.RemoveEmptyEntries ).ToHashSet() ?? new HashSet<string>();
          GameLogTextPaths.RemoveWhere( x => !x.StartsWith( "/" ) ); // clean up to ensure no 'empty' entries
          WhitespaceRemovalStrategy = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "WhitespaceRemovalStrategy", WhitespaceHandlingStrategy.TrimPerNewline );
+         RomajiPostProcessing = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "RomajiPostProcessing", RomajiPostProcessing.ReplaceMacronWithCircumflex | RomajiPostProcessing.RemoveApostrophes );
 
          TextureDirectory = PluginEnvironment.Current.Preferences.GetOrDefault( "Texture", "TextureDirectory", @"Translation\Texture" );
          EnableTextureTranslation = PluginEnvironment.Current.Preferences.GetOrDefault( "Texture", "EnableTextureTranslation", false );

src/XUnity.AutoTranslator.Plugin.Core/Endpoints/IInitializationContext.cs

@@ -33,7 +33,7 @@
       /// Disables the certificate check for the specified hostnames.
       /// </summary>
       /// <param name="hosts"></param>
-      void DisableCerfificateChecksFor( params string[] hosts );
+      void DisableCertificateChecksFor( params string[] hosts );
 
       /// <summary>
       /// Gets the source language that the plugin is configured with.

src/XUnity.AutoTranslator.Plugin.Core/Endpoints/InitializationContext.cs

@@ -35,7 +35,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
 
       public string PluginDirectory => PluginEnvironment.Current.DataPath;
 
-      public void DisableCerfificateChecksFor( params string[] hosts )
+      public void DisableCertificateChecksFor( params string[] hosts )
       {
          _security.EnableSslFor( hosts );
       }

src/XUnity.AutoTranslator.Plugin.Core/Extensions/IniFileExtensions.cs

@@ -1,6 +1,8 @@
 using System;
 using System.Globalization;
+using System.Linq;
 using ExIni;
+using XUnity.AutoTranslator.Plugin.Core.Utilities;
 
 namespace XUnity.AutoTranslator.Plugin.Core.Extensions
 {
@@ -22,7 +24,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Extensions
                {
                   if( typeOfT.IsEnum )
                   {
-                     iniKey.Value = Enum.GetName( typeOfT, defaultValue );
+                     iniKey.Value = EnumHelper.GetNames( typeOfT, defaultValue );
                   }
                   else
                   {
@@ -43,7 +45,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Extensions
                {
                   if( typeOfT.IsEnum )
                   {
-                     return (T)Enum.Parse( typeOfT, iniKey.Value, true );
+                     return (T)EnumHelper.GetValues( typeOfT, iniKey.Value );
                   }
                   else
                   {
@@ -61,7 +63,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Extensions
             {
                if( typeOfT.IsEnum )
                {
-                  iniKey.Value = Enum.GetName( typeOfT, defaultValue );
+                  iniKey.Value = EnumHelper.GetNames( typeOfT, defaultValue );
                }
                else
                {

src/XUnity.AutoTranslator.Plugin.Core/RomajiPostProcessing.cs

@@ -0,0 +1,16 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace XUnity.AutoTranslator.Plugin.Core
+{
+   [Flags]
+   internal enum RomajiPostProcessing
+   {
+      None                         = 0,
+      ReplaceMacronWithCircumflex  = 1 << 0,
+      RemoveAllDiacritics          = 1 << 1,
+      RemoveApostrophes            = 1 << 2
+   }
+}

src/XUnity.AutoTranslator.Plugin.Core/Utilities/EnumHelper.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace XUnity.AutoTranslator.Plugin.Core.Utilities
+{
+   internal static class EnumHelper
+   {
+      public static string GetNames( Type flagType, object value )
+      {
+         var attr = (FlagsAttribute)flagType.GetCustomAttributes( typeof( FlagsAttribute ), false ).FirstOrDefault();
+         if( attr == null )
+         {
+            return Enum.GetName( flagType, value );
+         }
+         else
+         {
+            var validEnumValues = Enum.GetValues( flagType );
+            var stringValues = Enum.GetNames( flagType );
+            var names = string.Empty;
+
+            foreach( var stringValue in stringValues )
+            {
+               var parsedEnumValue = Enum.Parse( flagType, stringValue, true );
+               foreach( var validEnumValue in validEnumValues )
+               {
+                  long validIntegerValue = Convert.ToInt64( validEnumValue );
+                  long integerValue = Convert.ToInt64( value );
+
+                  if( ( integerValue & validIntegerValue ) != 0 && Equals( parsedEnumValue, validEnumValue ) )
+                  {
+                     names += stringValue + ";";
+                     break;
+                  }
+               }
+            }
+
+            if( names.EndsWith( ";" ) )
+            {
+               names = names.Substring( 0, names.Length - 1 );
+            }
+
+            return names;
+         }
+      }
+
+      public static object GetValues( Type flagType, string commaSeparatedStringValue )
+      {
+         var attr = (FlagsAttribute)flagType.GetCustomAttributes( typeof( FlagsAttribute ), false ).FirstOrDefault();
+         if( attr == null )
+         {
+            return Enum.Parse( flagType, commaSeparatedStringValue, true );
+         }
+         else
+         {
+            var stringValues = commaSeparatedStringValue.Split( new char[] { ';', ' ' }, StringSplitOptions.RemoveEmptyEntries );
+            var validEnumValues = Enum.GetValues( flagType );
+            var underlyingType = Enum.GetUnderlyingType( flagType );
+            long flagValues = 0;
+
+            foreach( var stringValue in stringValues )
+            {
+               bool found = false;
+               foreach( var validEnumValue in validEnumValues )
+               {
+                  var validStringValue = Enum.GetName( flagType, validEnumValue );
+                  if( string.Equals( stringValue, validStringValue, StringComparison.OrdinalIgnoreCase ) )
+                  {
+                     var validInteger = Convert.ToInt64( validEnumValue );
+                     flagValues |= validInteger;
+                     found = true;
+                  }
+               }
+
+               if( !found )
+               {
+                  throw new ArgumentException( $"Requested value '{stringValue}' was not found." );
+               }
+            }
+            return Convert.ChangeType( flagValues, Enum.GetUnderlyingType( flagType ) );
+         }
+      }
+   }
+}

src/XUnity.AutoTranslator.Plugin.Core/Utilities/RomanizationHelper.cs

@@ -0,0 +1,137 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+
+namespace XUnity.AutoTranslator.Plugin.Core.Utilities
+{
+   /// <summary>
+   /// Helpers that ensures strings only contains standard ASCII characters.
+   /// </summary>
+   internal static class RomanizationHelper
+   {
+      public static string PostProcess( string text, RomajiPostProcessing postProcessing )
+      {
+         if( ( postProcessing & RomajiPostProcessing.ReplaceMacronWithCircumflex ) != 0 )
+         {
+            text = ConvertMacronToCircumflex( text );
+         }
+         if( ( postProcessing & RomajiPostProcessing.RemoveAllDiacritics ) != 0 )
+         {
+            text = RemoveAllDiacritics( text );
+         }
+         if( ( postProcessing & RomajiPostProcessing.ReplaceMacronWithCircumflex ) != 0 )
+         {
+            text = RemoveNApostrophe( text );
+         }
+         return text;
+      }
+
+      public static string ConvertMacronToCircumflex( string romanizedJapaneseText )
+      {
+         var builder = new StringBuilder( romanizedJapaneseText.Length );
+         for( int i = 0 ; i < romanizedJapaneseText.Length ; i++ )
+         {
+            var c = romanizedJapaneseText[ i ];
+
+            switch( c )
+            {
+               case 'Ā':
+                  builder.Append( 'Â' );
+                  break;
+               case 'ā':
+                  builder.Append( 'â' );
+                  break;
+               case 'Ī':
+                  builder.Append( 'Î' );
+                  break;
+               case 'ī':
+                  builder.Append( 'î' );
+                  break;
+               case 'Ū':
+                  builder.Append( 'Û' );
+                  break;
+               case 'ū':
+                  builder.Append( 'û' );
+                  break;
+               case 'Ē':
+                  builder.Append( 'Ê' );
+                  break;
+               case 'ē':
+                  builder.Append( 'ê' );
+                  break;
+               case 'Ō':
+                  builder.Append( 'Ô' );
+                  break;
+               case 'ō':
+                  builder.Append( 'ô' );
+                  break;
+               default:
+                  builder.Append( c );
+                  break;
+            }
+         }
+         return builder.ToString();
+      }
+
+      public static string RemoveNApostrophe( string romanizedJapaneseText )
+      {
+         return romanizedJapaneseText
+            .Replace( "n'", "n" )
+            .Replace( "n’", "n" );
+      }
+
+      /// <summary>
+      /// Remove diacritics (accents) from a string (for example ć -> c)
+      /// </summary>
+      /// <returns>ASCII compliant string</returns>
+      public static string RemoveAllDiacritics( this string input )
+      {
+         var text = input.SafeNormalize( NormalizationForm.FormD );
+         var chars = text.Where( c => CharUnicodeInfo.GetUnicodeCategory( c ) != UnicodeCategory.NonSpacingMark ).ToArray();
+         return new string( chars ).SafeNormalize( NormalizationForm.FormC );
+      }
+
+      /// <summary>
+      /// Safe version of normalize that doesn't crash on invalid code points in string.
+      /// Instead the points are replaced with question marks.
+      /// </summary>
+      private static string SafeNormalize( this string input, NormalizationForm normalizationForm = NormalizationForm.FormC )
+      {
+         return ReplaceNonCharacters( input, '?' ).Normalize( normalizationForm );
+      }
+
+      private static string ReplaceNonCharacters( string input, char replacement )
+      {
+         var sb = new StringBuilder( input.Length );
+         for( var i = 0 ; i < input.Length ; i++ )
+         {
+            if( char.IsSurrogatePair( input, i ) )
+            {
+               int c = char.ConvertToUtf32( input, i );
+               i++;
+               if( IsValidCodePoint( c ) )
+                  sb.Append( char.ConvertFromUtf32( c ) );
+               else
+                  sb.Append( replacement );
+            }
+            else
+            {
+               char c = input[ i ];
+               if( IsValidCodePoint( c ) )
+                  sb.Append( c );
+               else
+                  sb.Append( replacement );
+            }
+         }
+         return sb.ToString();
+      }
+
+      private static bool IsValidCodePoint( int point )
+      {
+         return point < 0xfdd0 || point >= 0xfdf0 && ( point & 0xffff ) != 0xffff && ( point & 0xfffe ) != 0xfffe && point <= 0x10ffff;
+      }
+   }
+}

src/XUnity.AutoTranslator.Plugin.Core/WhitespaceHandlingStrategy.cs

@@ -7,7 +7,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
 {
    internal enum WhitespaceHandlingStrategy
    {
-      AllOccurrences,
-      TrimPerNewline
+      TrimPerNewline,
+      AllOccurrences
    }
 }