Support for persistent HTTP connection by changing to WebClient over WWW

CHANGELOG.md

@@ -1,7 +1,10 @@
 ### 2.8.0
+ * CHANGE - Whether SSL is enabled or not is now entirely based on chosen endpoint support
  * FEATURE - Support for IMGUI translation texts with numbers
  * FEATURE - Support for overwriting IMGUI hook events
+ * BUG FIX - Improved fix for gtrans (23.07.2018) by supporting persistent HTTP connections and cookies and recalculation of TKK and SSL
  * BUG FIX - Fixed whitespace handling to honor configuration more appropriately
+ * MISC - Prints out to console errors that occurrs during translation
  * MISC - IMGUI is still disabled by default. Often other mods UI are implemented in IMGUI. Enabling it will allow those UIs to be translated as well. 
    * Simply change the config, such that: EnableIMGUI=True
 

src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs

@@ -98,7 +98,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
          LoadTranslations();
 
          // start a thread that will periodically removed unused references
-         var t1 = new Thread( RemovedUnusedReferences );
+         var t1 = new Thread( MaintenanceLoop );
          t1.IsBackground = true;
          t1.Start();
 
@@ -118,7 +118,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
             .ToArray();
       }
 
-      private void RemovedUnusedReferences( object state )
+      private void MaintenanceLoop( object state )
       {
          while( true )
          {
@@ -130,10 +130,8 @@ namespace XUnity.AutoTranslator.Plugin.Core
             {
                Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: An unexpected error occurred while removing GC'ed resources." + Environment.NewLine + e );
             }
-            finally
-            {
-               Thread.Sleep( 1000 * 60 );
-            }
+
+            Thread.Sleep( 1000 * 60 );
          }
       }
 
@@ -660,6 +658,8 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
          try
          {
+            UnityWebClient.CleanupDefault();
+
             CopyToClipboard();
             ResetThresholdTimerIfRequired();
 

src/XUnity.AutoTranslator.Plugin.Core/Configuration/Settings.cs

@@ -40,7 +40,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
       public static bool AllowPluginHookOverride;
       public static bool IgnoreWhitespaceInDialogue;
       public static int MinDialogueChars;
-      public static bool EnableSSL;
       public static string BaiduAppId;
       public static string BaiduAppSecret;
       public static int ForceSplitTextAfterCharacters;
@@ -60,6 +59,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
             }
 
             Config.Current.Preferences.DeleteSection( "AutoTranslator" );
+            Config.Current.Preferences[ "Service" ].DeleteKey( "EnableSSL" );
          }
          catch( Exception e )
          {
@@ -69,7 +69,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Configuration
 
 
          ServiceEndpoint = Config.Current.Preferences[ "Service" ][ "Endpoint" ].GetOrDefault( KnownEndpointNames.GoogleTranslate, true );
-         EnableSSL = Config.Current.Preferences[ "Service" ][ "EnableSSL" ].GetOrDefault( false );
 
          Language = Config.Current.Preferences[ "General" ][ "Language" ].GetOrDefault( "en" );
          FromLanguage = Config.Current.Preferences[ "General" ][ "FromLanguage" ].GetOrDefault( "ja", true );

src/XUnity.AutoTranslator.Plugin.Core/Constants/Types.cs

@@ -18,8 +18,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Constants
 
       public static readonly Type UILabel = FindType( "UILabel" );
 
-      public static readonly Type WWW = FindType( "UnityEngine.WWW" );
-
       private static Type FindType( string name )
       {
          return AppDomain.CurrentDomain.GetAssemblies()

src/XUnity.AutoTranslator.Plugin.Core/Web/AutoTranslateClient.cs

@@ -16,21 +16,20 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 {
    public static class AutoTranslateClient
    {
-      private static readonly ConstructorInfo WwwConstructor = Types.WWW.GetConstructor( new[] { typeof( string ), typeof( byte[] ), typeof( Dictionary<string, string> ) } );
-
-      private static KnownEndpoint _endpoint;
+      public static KnownEndpoint Endpoint;
       private static int _runningTranslations = 0;
       private static int _translationCount;
       private static int _maxConcurrency;
+      private static bool _isSettingUp = false;
 
       public static void Configure()
       {
-         _endpoint = KnownEndpoints.FindEndpoint( Settings.ServiceEndpoint );
-         _maxConcurrency = _endpoint.GetMaxConcurrency();
+         Endpoint = KnownEndpoints.FindEndpoint( Settings.ServiceEndpoint );
+         _maxConcurrency = 1; // WebClient does not support concurrency
 
-         if( _endpoint != null )
+         if( Endpoint != null )
          {
-            _endpoint.ConfigureServicePointManager();
+            Endpoint.ConfigureServicePointManager();
          }
       }
 
@@ -38,29 +37,41 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
       {
          get
          {
-            return _endpoint != null;
+            return Endpoint != null;
          }
       }
 
-      public static bool HasAvailableClients => !_endpoint.IsSettingUp() && _runningTranslations < _maxConcurrency;
+      public static bool HasAvailableClients => !_isSettingUp && _runningTranslations < _maxConcurrency;
 
       public static bool Fallback()
       {
-         return _endpoint.Fallback();
+         return Endpoint.Fallback();
       }
 
       public static IEnumerator TranslateByWWW( string untranslated, string from, string to, Action<string> success, Action failure )
       {
-         // allow self setup of async acquired info by an endpoint
-         var yieldable = _endpoint.StartSetup();
-         if( yieldable != null )
+         UnityWebClient.TouchedDefault();
+
+         try
          {
-            yield return yieldable;
-            _endpoint.EndSetup( yieldable );
+            _isSettingUp = true;
+
+            var setup = Endpoint.Setup( _translationCount );
+            if( setup != null )
+            {
+               while( setup.MoveNext() )
+               {
+                  yield return setup.Current;
+               }
+            }
+         }
+         finally
+         {
+            _isSettingUp = false;
          }
 
-         var url = _endpoint.GetServiceUrl( untranslated, from, to );
-         _endpoint.ApplyHeaders( UnityWebClient.Default.Headers );
+         var url = Endpoint.GetServiceUrl( untranslated, from, to );
+         Endpoint.ApplyHeaders( UnityWebClient.Default.Headers );
          var result = UnityWebClient.Default.GetDownloadResult( new Uri( url ) );
          try
          {
@@ -89,32 +100,29 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
                }
             }
 
-            if( _translationCount % 100 == 0 )
-            {
-               UnityWebClient.Default.ClearCookies();
-            }
-
 
             if( result.Succeeded )
             {
-               if( _endpoint.TryExtractTranslated( result.Result, out var translatedText ) )
+               if( Endpoint.TryExtractTranslated( result.Result, out var translatedText ) )
                {
                   translatedText = translatedText ?? string.Empty;
                   success( translatedText );
                }
                else
                {
+                  Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: Error occurred while extracting translation." );
                   failure();
                }
             }
             else
             {
+               Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: Error occurred while retrieving translation." + Environment.NewLine + result.Error );
                failure();
             }
          }
          finally
          {
-
+            UnityWebClient.TouchedDefault();
          }
       }
    }

src/XUnity.AutoTranslator.Plugin.Core/Web/BaiduTranslateEndpoint.cs

@@ -16,7 +16,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
    {
       private static ServicePoint ServicePoint;
       private static readonly string HttpServicePointTemplateUrl = "http://api.fanyi.baidu.com/api/trans/vip/translate?q={0}&from={1}&to={2}&appid={3}&salt={4}&sign={5}";
-      private static readonly string HttpsServicePointTemplateUrl = "https://api.fanyi.baidu.com/api/trans/vip/translate?q={0}&from={1}&to={2}&appid={3}&salt={4}&sign={5}";
 
       private static readonly MD5 HashMD5 = MD5.Create();
 
@@ -53,14 +52,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 
       public override void ConfigureServicePointManager()
       {
-         try
-         {
-            ServicePoint = ServicePointManager.FindServicePoint( new Uri( "http://api.fanyi.baidu.com" ) );
-            ServicePoint.ConnectionLimit = GetMaxConcurrency();
-         }
-         catch
-         {
-         }
       }
 
       public override bool TryExtractTranslated( string result, out string translated )
@@ -102,7 +93,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
          string salt = DateTime.UtcNow.Millisecond.ToString();
          var md5 = CreateMD5( Settings.BaiduAppId + untranslatedText + salt + Settings.BaiduAppSecret );
 
-         return string.Format( Settings.EnableSSL ? HttpsServicePointTemplateUrl : HttpServicePointTemplateUrl, WWW.EscapeURL( untranslatedText ), from, to, Settings.BaiduAppId, salt, md5 );
+         return string.Format( HttpServicePointTemplateUrl, WWW.EscapeURL( untranslatedText ), from, to, Settings.BaiduAppId, salt, md5 );
       }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Web/DefaultEndpoint.cs

@@ -26,14 +26,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 
       public override void ConfigureServicePointManager()
       {
-         try
-         {
-            ServicePoint = ServicePointManager.FindServicePoint( new Uri( Identifier ) );
-            ServicePoint.ConnectionLimit = GetMaxConcurrency();
-         }
-         catch
-         {
-         }
       }
 
       public override bool TryExtractTranslated( string result, out string translated )
@@ -46,10 +38,5 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
       {
          return string.Format( ServicePointTemplateUrl, Identifier, from, to, WWW.EscapeURL( untranslatedText ) );
       }
-
-      public override int GetMaxConcurrency()
-      {
-         return 10;
-      }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Web/GoogleTranslateEndpoint.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Globalization;
 using System.IO;
@@ -17,19 +18,13 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 {
    public class GoogleTranslateEndpoint : KnownEndpoint
    {
-      private static readonly ConstructorInfo WwwConstructor = Constants.Types.WWW.GetConstructor( new[] { typeof( string ), typeof( byte[] ), typeof( Dictionary<string, string> ) } );
-
       private static readonly string CertificateIssuer = "CN=Google Internet Authority G3, O=Google Trust Services, C=US";
-      private static ServicePoint ServicePoint;
-      private static readonly string HttpServicePointTemplateUrl = "http://translate.googleapis.com/translate_a/single?client=t&dt=t&sl={0}&tl={1}&ie=UTF-8&oe=UTF-8&tk={2}&q={3}";
       private static readonly string HttpsServicePointTemplateUrl = "https://translate.googleapis.com/translate_a/single?client=t&dt=t&sl={0}&tl={1}&ie=UTF-8&oe=UTF-8&tk={2}&q={3}";
-      private static readonly string FallbackHttpServicePointTemplateUrl = "http://translate.googleapis.com/translate_a/single?client=gtx&sl={0}&tl={1}&dt=t&q={2}";
       private static readonly string FallbackHttpsServicePointTemplateUrl = "https://translate.googleapis.com/translate_a/single?client=gtx&sl={0}&tl={1}&dt=t&q={2}";
       private static readonly string HttpsTranslateUserSite = "https://translate.google.com";
-      private static readonly string HttpTranslateUserSite = "http://translate.google.com";
 
+      private CookieContainer _cookieContainer;
       private bool _hasFallenBack = false;
-      private bool _isSettingUp = false;
       private bool _hasSetup = false;
       private long m = 425635;
       private long s = 1953544246;
@@ -37,7 +32,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
       public GoogleTranslateEndpoint()
          : base( KnownEndpointNames.GoogleTranslate )
       {
-
+         _cookieContainer = new CookieContainer();
       }
 
       public override void ApplyHeaders( Dictionary<string, string> headers )
@@ -52,80 +47,79 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
          headers[ HttpRequestHeader.Accept ] = "*/*";
       }
 
-      public override bool IsSettingUp()
-      {
-         return _isSettingUp;
-      }
-
-      public override object StartSetup()
+      public override IEnumerator Setup( int translationCount )
       {
-         if( !_isSettingUp && !_hasSetup )
+         if( !_hasSetup || translationCount % 100 == 0 )
          {
-            _isSettingUp = true;
+            _hasSetup = true;
+            // Setup TKK and cookies
 
-            try
-            {
-               ApplyHeaders( UnityWebClient.Default.Headers );
-               var result = UnityWebClient.Default.GetDownloadResult( new Uri( Settings.EnableSSL ? HttpsTranslateUserSite : HttpTranslateUserSite ) );
-               return result;
-            }
-            catch( Exception e )
-            {
-               _hasSetup = true;
-               _isSettingUp = false;
+            return SetupTKK();
 
-               Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: An error occurred while setting up GoogleTranslate TKK (Start)." + Environment.NewLine + e.ToString() );
-
-               return null;
-            }
          }
-         return null;
+         else
+         {
+            return null;
+         }
       }
 
-      public override void EndSetup( object www )
+      public IEnumerator SetupTKK()
       {
-         Console.WriteLine( "EndSetup: " + www );
+         string error = null;
+         DownloadResult downloadResult = null;
 
-         var dresult = www as DownloadResult;
-         var error = dresult.Error;
+         _cookieContainer = new CookieContainer();
 
-         if( dresult.Succeeded )
+         try
          {
-            try
-            {
-               var html = dresult.Result;
-
-               const string lookup = "TKK=eval('";
-               var lookupIndex = html.IndexOf( lookup ) + lookup.Length;
-               var openClamIndex = html.IndexOf( '{', lookupIndex );
-               var closeClamIndex = html.IndexOf( '}', openClamIndex );
-               var functionIndex = html.IndexOf( "function", lookupIndex );
-               var script = html.Substring( functionIndex, closeClamIndex - functionIndex + 1 );
-               var decodedScript = script.Replace( "\\x3d", "=" ).Replace( "\\x27", "'" ).Replace( "function", "function FuncName" );
-
-               // https://github.com/paulbartrum/jurassic/wiki/Safely-executing-user-provided-scripts
-               ScriptEngine engine = new ScriptEngine();
-               engine.Evaluate( decodedScript );
-               var result = engine.CallGlobalFunction<string>( "FuncName" );
-
-               var parts = result.Split( '.' );
-               m = long.Parse( parts[ 0 ] );
-               s = long.Parse( parts[ 1 ] );
-
-               Console.WriteLine( "[XUnity.AutoTranslator][INFO]: Successfully setup GoogleTranslate endpoint." );
-            }
-            catch( Exception e )
+            ApplyHeaders( UnityWebClient.Default.Headers );
+            downloadResult = UnityWebClient.Default.GetDownloadResult( new Uri( HttpsTranslateUserSite ) );
+         }
+         catch( Exception e )
+         {
+            error = e.ToString();
+         }
+
+         if( downloadResult != null )
+         {
+            yield return downloadResult;
+
+            error = downloadResult.Error;
+            if( downloadResult.Succeeded )
             {
-               error = e.ToString();
+               try
+               {
+                  var html = downloadResult.Result;
+
+                  const string lookup = "TKK=eval('";
+                  var lookupIndex = html.IndexOf( lookup ) + lookup.Length;
+                  var openClamIndex = html.IndexOf( '{', lookupIndex );
+                  var closeClamIndex = html.IndexOf( '}', openClamIndex );
+                  var functionIndex = html.IndexOf( "function", lookupIndex );
+                  var script = html.Substring( functionIndex, closeClamIndex - functionIndex + 1 );
+                  var decodedScript = script.Replace( "\\x3d", "=" ).Replace( "\\x27", "'" ).Replace( "function", "function FuncName" );
+
+                  // https://github.com/paulbartrum/jurassic/wiki/Safely-executing-user-provided-scripts
+                  ScriptEngine engine = new ScriptEngine();
+                  engine.Evaluate( decodedScript );
+                  var result = engine.CallGlobalFunction<string>( "FuncName" );
+
+                  var parts = result.Split( '.' );
+                  m = long.Parse( parts[ 0 ] );
+                  s = long.Parse( parts[ 1 ] );
+
+                  Console.WriteLine( "[XUnity.AutoTranslator][INFO]: Successfully setup GoogleTranslate TKK." );
+               }
+               catch( Exception e )
+               {
+                  error = e.ToString();
+               }
             }
          }
 
-         _hasSetup = true;
-         _isSettingUp = false;
-
          if( error != null )
          {
-            Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: An error occurred while setting up GoogleTranslate TKK (End)." + Environment.NewLine + error );
+            Console.WriteLine( "[XUnity.AutoTranslator][ERROR]: An error occurred while setting up GoogleTranslate TKK." + Environment.NewLine + error );
          }
       }
 
@@ -202,9 +196,6 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
                return certificate.Issuer == CertificateIssuer;
             };
 
-            ServicePoint = ServicePointManager.FindServicePoint( new Uri( "http://translate.googleapis.com" ) );
-            ServicePoint.ConnectionLimit = GetMaxConcurrency();
-
          }
          catch
          {
@@ -242,11 +233,11 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
       {
          if( _hasFallenBack )
          {
-            return string.Format( Settings.EnableSSL ? FallbackHttpsServicePointTemplateUrl : FallbackHttpServicePointTemplateUrl, from, to, WWW.EscapeURL( untranslatedText ) );
+            return string.Format( FallbackHttpsServicePointTemplateUrl, from, to, WWW.EscapeURL( untranslatedText ) );
          }
          else
          {
-            return string.Format( Settings.EnableSSL ? HttpsServicePointTemplateUrl : HttpServicePointTemplateUrl, from, to, Tk( untranslatedText ), WWW.EscapeURL( untranslatedText ) );
+            return string.Format( HttpsServicePointTemplateUrl, from, to, Tk( untranslatedText ), WWW.EscapeURL( untranslatedText ) );
          }
       }
 
@@ -260,5 +251,22 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 
          return false;
       }
+
+      public override void WriteCookies( HttpWebResponse response )
+      {
+         CookieCollection cookies = response.Cookies;
+         foreach( Cookie cookie in cookies )
+         {
+            // redirect cookie to correct domain
+            cookie.Domain = ".googleapis.com";
+         }
+
+         _cookieContainer.Add( cookies );
+      }
+
+      public override CookieContainer ReadCookies()
+      {
+         return _cookieContainer;
+      }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Web/KnownEndpoint.cs

@@ -1,4 +1,5 @@
 using System;
+using System.Collections;
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
@@ -25,29 +26,24 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
 
       public abstract bool TryExtractTranslated( string result, out string translated );
 
-      public virtual bool Fallback()
+      public virtual void WriteCookies( HttpWebResponse response )
       {
-         return false;
+
       }
 
-      public virtual int GetMaxConcurrency()
+      public virtual CookieContainer ReadCookies()
       {
-         return 1;
+         return null;
       }
 
-      public virtual bool IsSettingUp()
+      public virtual bool Fallback()
       {
          return false;
       }
 
-      public virtual object StartSetup()
+      public virtual IEnumerator Setup( int translationCount )
       {
          return null;
       }
-
-      public virtual void EndSetup( object obj )
-      {
-
-      }
    }
 }

src/XUnity.AutoTranslator.Plugin.Core/Web/UnityWebClient.cs

@@ -1,15 +1,22 @@
 using System;
 using System.Collections;
 using System.Collections.Generic;
+using System.Collections.Specialized;
+using System.IO;
 using System.Linq;
 using System.Net;
+using System.Reflection;
 using System.Text;
+using Harmony;
 using UnityEngine;
 
 namespace XUnity.AutoTranslator.Plugin.Core.Web
 {
    public class UnityWebClient : WebClient
    {
+      private static readonly TimeSpan MaxUnusedLifespan = TimeSpan.FromSeconds( 20 );
+
+      private static DateTime _defaultLastUse = DateTime.UtcNow;
       private static UnityWebClient _default;
 
       public static UnityWebClient Default
@@ -19,22 +26,31 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
             if( _default == null )
             {
                _default = new UnityWebClient();
+               TouchedDefault();
             }
             return _default;
          }
       }
 
-      private CookieContainer _container = new CookieContainer();
+      public static void TouchedDefault()
+      {
+         _defaultLastUse = DateTime.UtcNow;
+      }
 
-      public UnityWebClient()
+      public static void CleanupDefault()
       {
-         Encoding = Encoding.UTF8;
-         DownloadStringCompleted += UnityWebClient_DownloadStringCompleted;
+         var client = _default;
+         if( client != null && DateTime.UtcNow - _defaultLastUse > MaxUnusedLifespan )
+         {
+            _default = null;
+            client.Dispose();
+         }
       }
 
-      public void ClearCookies()
+      public UnityWebClient()
       {
-         _container = new CookieContainer();
+         Encoding = Encoding.UTF8;
+         DownloadStringCompleted += UnityWebClient_DownloadStringCompleted;
       }
 
       private void UnityWebClient_DownloadStringCompleted( object sender, DownloadStringCompletedEventArgs ev )
@@ -70,19 +86,39 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
          var httpRequest = request as HttpWebRequest;
          if( httpRequest != null )
          {
-            httpRequest.CookieContainer = _container;
+            var cookies = AutoTranslateClient.Endpoint.ReadCookies();
+            httpRequest.CookieContainer = cookies;
          }
          return request;
       }
 
-      public DownloadResult GetDownloadResult( Uri address )
+      protected override WebResponse GetWebResponse( WebRequest request, IAsyncResult result )
       {
-         var handle = new DownloadResult();
+         WebResponse response = base.GetWebResponse( request, result );
+         WriteCookies( response );
+         return response;
+      }
 
-         DownloadStringAsync( address, handle );
+      protected override WebResponse GetWebResponse( WebRequest request )
+      {
+         WebResponse response = base.GetWebResponse( request );
+         WriteCookies( response );
+         return response;
+      }
 
-         Console.WriteLine( "GetDownloadResult: " + address );
+      private void WriteCookies( WebResponse r )
+      {
+         var response = r as HttpWebResponse;
+         if( response != null )
+         {
+            AutoTranslateClient.Endpoint.WriteCookies( response );
+         }
+      }
 
+      public DownloadResult GetDownloadResult( Uri address )
+      {
+         var handle = new DownloadResult();
+         DownloadStringAsync( address, handle );
          return handle;
       }
    }