more settings to control caching

CHANGELOG.md

@@ -1,4 +1,8 @@
-### 3.3.0
+### 3.3.1
+ * MISC - Options to cache results of regex lookups and whitespace differences
+ * BUG FIX - Fully clear translations before reloading (ALT+R)
+
+### 3.3.0
  * FEATURE - Support 'TARC' regex formatting in translation files
  * FEATURE - Much improved handling of whitespace and newlines. Option 'TrimAllText' removed and options for 'WhitespaceRemovalStrategy' changed
  * BUG FIX - Allow hooking of text with components named 'Dummy'

src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs

@@ -376,7 +376,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
          var added = endpoint.EnqueueTranslation( ui, key, translationResult, context );
          if( added && translationResult == null )
          {
-            SpamChecker.PerformChecks( key.TrimmedText );
+            SpamChecker.PerformChecks( key.TrimmedTranslatableText );
          }
       }
 
@@ -433,10 +433,10 @@ namespace XUnity.AutoTranslator.Plugin.Core
       {
          if( Settings.CopyToClipboard && Features.SupportsClipboard )
          {
-            if( !_textsToCopyToClipboard.Contains( key.TrimmedText ) )
+            if( !_textsToCopyToClipboard.Contains( key.TrimmedTranslatableText ) )
             {
-               _textsToCopyToClipboard.Add( key.TrimmedText );
-               _textsToCopyToClipboardOrdered.Add( key.TrimmedText );
+               _textsToCopyToClipboard.Add( key.TrimmedTranslatableText );
+               _textsToCopyToClipboardOrdered.Add( key.TrimmedTranslatableText );
 
                _clipboardUpdated = Time.realtimeSinceStartup;
             }
@@ -1397,7 +1397,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
       /// </summary>
       private IEnumerator WaitForTextStablization( UntranslatedText textKey, float delay, Action onTextStabilized, Action onFailed = null )
       {
-         var text = textKey.TrimmedText;
+         var text = textKey.TrimmedTranslatableText;
 
          if( !_immediatelyTranslating.Contains( text ) )
          {
@@ -1668,7 +1668,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
             TextCache.AddTranslationToCache( job.Key.TranslatableText, job.TranslatedText );
          }
 
-         job.Endpoint.AddTranslationToCache( job.Key, job.TranslatedText );
+         job.Endpoint.AddTranslationToCache( job.Key.TranslatableText, job.TranslatedText );
 
          // fix translation results directly on jobs
          foreach( var translationResult in job.TranslationResults )

src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs

@@ -88,6 +88,8 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
       public static TextPostProcessing TranslationPostProcessing;
       public static bool EnableExperimentalHooks;
       public static bool ForceExperimentalHooks;
+      public static bool CacheRegexLookups;
+      public static bool CacheWhitespaceDifferences;
 
       public static string TextureDirectory;
       public static bool EnableTextureTranslation;
@@ -146,6 +148,8 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
             TranslationPostProcessing = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "TranslationPostProcessing", TextPostProcessing.ReplaceMacronWithCircumflex );
             EnableExperimentalHooks = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "EnableExperimentalHooks", false );
             ForceExperimentalHooks = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "ForceExperimentalHooks", false );
+            CacheRegexLookups = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "CacheRegexLookups", true );
+            CacheWhitespaceDifferences = PluginEnvironment.Current.Preferences.GetOrDefault( "Behaviour", "CacheWhitespaceDifferences", true );
 
             TextureDirectory = PluginEnvironment.Current.Preferences.GetOrDefault( "Texture", "TextureDirectory", @"Translation\Texture" );
             EnableTextureTranslation = PluginEnvironment.Current.Preferences.GetOrDefault( "Texture", "EnableTextureTranslation", false );

src/XUnity.AutoTranslator.Plugin.Core/Endpoints/TranslationEndpointManager.cs

@@ -70,7 +70,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
             return result;
          }
 
-         var modifiedKey = key.TrimmedText;
+         var modifiedKey = key.TrimmedTranslatableText;
          result = _translations.TryGetValue( modifiedKey, out value );
          if( result )
          {
@@ -96,12 +96,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
          // FIXME: Implement
       }
 
-      public void AddTranslationToCache( UntranslatedText key, string value )
-      {
-         // UNRELEASED: Not included in current release
-         //AddTranslationToCache( key.GetDictionaryLookupKey(), value );
-      }
-
       public void AddTranslationToCache( string key, string value )
       {
          // UNRELEASED: Not included in current release
@@ -142,7 +136,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
                _unstartedJobs.Remove( key );
                Manager.UnstartedTranslations--;
 
-               var untranslatedText = job.Key.TrimmedText;
+               var untranslatedText = job.Key.TrimmedTranslatableText;
                if( CanTranslate( untranslatedText ) )
                {
                   jobs.Add( job );
@@ -199,7 +193,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
             _unstartedJobs.Remove( key );
             Manager.UnstartedTranslations--;
 
-            var untranslatedText = job.Key.TrimmedText;
+            var untranslatedText = job.Key.TrimmedTranslatableText;
             if( CanTranslate( untranslatedText ) )
             {
                _ongoingJobs[ key ] = job;
@@ -249,7 +243,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
 
                RemoveOngoingTranslation( job.Key );
 
-               XuaLogger.Current.Info( $"Completed: '{job.Key.TrimmedText}' => '{job.TranslatedText}'" );
+               XuaLogger.Current.Info( $"Completed: '{job.Key.TrimmedTranslatableText}' => '{job.TranslatedText}'" );
 
                Manager.InvokeJobCompleted( job );
             }
@@ -286,7 +280,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
 
          RemoveOngoingTranslation( job.Key );
 
-         XuaLogger.Current.Info( $"Completed: '{job.Key.TrimmedText}' => '{job.TranslatedText}'" );
+         XuaLogger.Current.Info( $"Completed: '{job.Key.TrimmedTranslatableText}' => '{job.TranslatedText}'" );
 
          Manager.InvokeJobCompleted( job );
       }
@@ -338,7 +332,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
 
                RemoveOngoingTranslation( key );
 
-               RegisterTranslationFailureFor( key.TrimmedText );
+               RegisterTranslationFailureFor( key.TrimmedTranslatableText );
 
                Manager.InvokeJobFailed( job );
             }
@@ -411,7 +405,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
             }
          }
 
-         XuaLogger.Current.Debug( "Queued: '" + key.TrimmedText + "'" );
+         XuaLogger.Current.Debug( "Queued: '" + key.TrimmedTranslatableText + "'" );
 
          var saveResultGlobally = checkOtherEndpoints;
          var newJob = new TranslationJob( this, key, saveResultGlobally );
@@ -476,7 +470,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints
 
          foreach( var job in unstartedJobs )
          {
-            XuaLogger.Current.Warn( $"Dequeued: '{job.Key.TrimmedText}'" );
+            XuaLogger.Current.Warn( $"Dequeued: '{job.Key.TrimmedTranslatableText}'" );
             job.Value.State = TranslationJobState.Failed;
             job.Value.ErrorMessage = "Translation failed because all jobs on endpoint was cleared.";
 

src/XUnity.AutoTranslator.Plugin.Core/Hooks/IMGUIHooks.cs

@@ -27,7 +27,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Hooks.IMGUI
          typeof( GUI_DoButton_Hook ),
          typeof( GUI_DoModalWindow_Hook ),
          typeof( GUI_DoWindow_Hook ),
-         typeof( GUI_DoTextField_Hook ),
+         //typeof( GUI_DoTextField_Hook ), // Why did I think this was a good idea?
          typeof( GUI_DoButtonGrid_Hook ),
          typeof( GUI_DoToggle_Hook ),
       };

src/XUnity.AutoTranslator.Plugin.Core/TextTranslationCache.cs

@@ -53,6 +53,8 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
                _registeredRegexes.Clear();
                _defaultRegexes.Clear();
+               _translations.Clear();
+               _reverseTranslations.Clear();
 
                var mainTranslationFile = Settings.AutoTranslationsFilePath;
                LoadTranslationsInFile( mainTranslationFile );
@@ -110,9 +112,9 @@ namespace XUnity.AutoTranslator.Plugin.Core
                            // also add a modified version of the translation
                            var ukey = new UntranslatedText( key, false, false );
                            var uvalue = new UntranslatedText( value, false, false );
-                           if( ukey.TrimmedText != key )
+                           if( ukey.TrimmedTranslatableText != key )
                            {
-                              AddTranslation( ukey.TrimmedText, uvalue.TrimmedText );
+                              AddTranslation( ukey.TrimmedTranslatableText, uvalue.TrimmedTranslatableText );
                            }
                         }
                         break;
@@ -220,6 +222,15 @@ namespace XUnity.AutoTranslator.Plugin.Core
          if( !HasTranslated( key ) )
          {
             AddTranslation( key, value );
+
+            // also add a trimmed version of the translation
+            var ukey = new UntranslatedText( key, false, false );
+            var uvalue = new UntranslatedText( value, false, false );
+            if( ukey.TrimmedTranslatableText != key )
+            {
+               AddTranslation( ukey.TrimmedTranslatableText, uvalue.TrimmedTranslatableText );
+            }
+
             if( persistToDisk )
             {
                QueueNewTranslationForDisk( key, value );
@@ -236,40 +247,51 @@ namespace XUnity.AutoTranslator.Plugin.Core
             return result;
          }
 
-         var modifiedKey = key.TrimmedText;
-         result = _translations.TryGetValue( modifiedKey, out value );
-         if( result )
+         var modifiedKey = key.TrimmedTranslatableText;
+         if( modifiedKey != unmodifiedKey )
          {
-            // add an unmodifiedKey to the dictionary
-            var unmodifiedValue = key.LeadingWhitespace + value + key.TrailingWhitespace;
+            result = _translations.TryGetValue( modifiedKey, out value );
+            if( result )
+            {
+               // add an unmodifiedKey to the dictionary
+               var unmodifiedValue = key.LeadingWhitespace + value + key.TrailingWhitespace;
 
-            XuaLogger.Current.Info( $"Whitespace difference: '{key.TrimmedText}' => '{value}'" );
-            AddTranslationToCache( unmodifiedKey, unmodifiedValue, false );
+               XuaLogger.Current.Info( $"Whitespace difference: '{key.TrimmedTranslatableText}' => '{value}'" );
+               AddTranslationToCache( unmodifiedKey, unmodifiedValue, Settings.CacheWhitespaceDifferences );
 
-            value = unmodifiedValue;
-            return result;
+               value = unmodifiedValue;
+               return result;
+            }
          }
 
          if( allowRegex )
          {
             bool found = false;
 
-            var len = _defaultRegexes.Count;
-            for( int i = 0; i < len; i++ )
+            for( int i = _defaultRegexes.Count - 1; i > -1; i-- )
             {
                var regex = _defaultRegexes[ i ];
-               var match = regex.CompiledRegex.Match( unmodifiedKey );
-               if( !match.Success ) continue;
+               try
+               {
+                  var match = regex.CompiledRegex.Match( unmodifiedKey );
+                  if( !match.Success ) continue;
 
-               var translation = regex.CompiledRegex.Replace( unmodifiedKey, regex.Translation );
+                  var translation = regex.CompiledRegex.Replace( unmodifiedKey, regex.Translation );
 
-               AddTranslationToCache( unmodifiedKey, translation, false ); // Would store it to file... Should we????
+                  AddTranslationToCache( unmodifiedKey, translation, Settings.CacheRegexLookups ); // Would store it to file... Should we????
 
-               value = translation;
-               found = true;
+                  value = translation;
+                  found = true;
 
-               XuaLogger.Current.Info( $"Regex lookup: '{key.TrimmedText}' => '{value}'" );
-               break;
+                  XuaLogger.Current.Info( $"Regex lookup: '{key.TrimmedTranslatableText}' => '{value}'" );
+                  break;
+               }
+               catch( Exception e )
+               {
+                  _defaultRegexes.RemoveAt( i );
+
+                  XuaLogger.Current.Error( e, $"Failed while attempting to replace or match text of regex '{regex.Original}'. Removing that regex from the cache." );
+               }
             }
 
             if( found )

src/XUnity.AutoTranslator.Plugin.Core/UI/AggregatedTranslationViewModel.cs

@@ -39,7 +39,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.UI
                if( _started.HasValue )
                {
                   var timeSince = Time.realtimeSinceStartup - _started.Value;
-                  if( timeSince > 1.0f )
+                  if( timeSince > 2.0f )
                   {
                      foreach( var additionTranslation in AggregatedTranslations )
                      {

src/XUnity.AutoTranslator.Plugin.Core/UntranslatedText.cs

@@ -134,11 +134,11 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
          if( leadingWhitespaceCount > 0 || trailingWhitespaceCount > 0 )
          {
-            TrimmedText = TranslatableText.Substring( leadingWhitespaceCount, TranslatableText.Length - trailingWhitespaceCount - leadingWhitespaceCount );
+            TrimmedTranslatableText = TranslatableText.Substring( leadingWhitespaceCount, TranslatableText.Length - trailingWhitespaceCount - leadingWhitespaceCount );
          }
          else
          {
-            TrimmedText = TranslatableText;
+            TrimmedTranslatableText = TranslatableText;
          }
       }
 
@@ -146,10 +146,10 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
       public string TrailingWhitespace { get; }
 
-      public string TrimmedText { get; }
-
       public string TranslatableText { get; }
 
+      public string TrimmedTranslatableText { get; }
+
       public string OriginalText { get; }
 
       public TemplatedString TemplatedText { get; }

test/XUnity.AutoTranslator.Plugin.Core.Tests/UntranslatedTextTests.cs

@@ -19,7 +19,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Tests
          var untranslatedText = new UntranslatedText( input, false, false );
 
          Assert.Equal( input, untranslatedText.TranslatableText );
-         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedText );
+         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedTranslatableText );
          Assert.Equal( expectedLeadingWhitespace, untranslatedText.LeadingWhitespace );
          Assert.Equal( expectedTrailingWhitespace, untranslatedText.TrailingWhitespace );
       }
@@ -31,7 +31,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Tests
       {
          var untranslatedText = new UntranslatedText( input, false, true );
          
-         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedText );
+         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedTranslatableText );
       }
 
       [Theory( DisplayName = "Can_Trim_Internal_And_Surrounding_Whitespace" )]
@@ -42,7 +42,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Tests
       {
          var untranslatedText = new UntranslatedText( input, false, true );
 
-         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedText );
+         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedTranslatableText );
          Assert.Equal( expectedLeadingWhitespace, untranslatedText.LeadingWhitespace );
          Assert.Equal( expectedTrailingWhitespace, untranslatedText.TrailingWhitespace );
       }
@@ -53,7 +53,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Tests
       {
          var untranslatedText = new UntranslatedText( input, true, true );
 
-         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedText );
+         Assert.Equal( expectedTrimmedText, untranslatedText.TrimmedTranslatableText );
          Assert.Equal( expectedLeadingWhitespace, untranslatedText.LeadingWhitespace );
          Assert.Equal( expectedTrailingWhitespace, untranslatedText.TrailingWhitespace );
       }