user agents fix, super fix watson google legit batching

CHANGELOG.md

@@ -7,6 +7,7 @@
  * FEATURE - Removed support for Excite translate because it only support the 'WWW' API in Unity due to missing TLS1.2 support
  * FEATURE - Updated Watson translate to v3
  * FEATURE - Support for 'romaji' as output language. Only google supports this at the moment
+ * FEATURE - Batching support for all endpoints where the API supports it
  * BUG FIX - Too many small fixes to mention
  * MISC - {GameExeName} variable can now be used in configuration of directories and files
  * MISC - Changed the way the 'Custom' endpoint works. See README for more info

README.md

@@ -600,8 +600,7 @@ internal class YandexTranslateEndpoint : HttpEndpoint
             context.DestinationLanguage,
             WwwHelper.EscapeUrl( context.UntranslatedText ),
             _key ) );
-
-      request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.183 Safari/537.36 Vivaldi/1.96.1147.55" : AutoTranslatorSettings.UserAgent;
+         
       request.Headers[ HttpRequestHeader.Accept ] = "*/*";
       request.Headers[ HttpRequestHeader.AcceptCharset ] = "UTF-8";
 

src/Translators/BaiduTranslate/BaiduTranslateEndpoint.cs

@@ -4,6 +4,7 @@ using System.Security.Cryptography;
 using System.Text;
 using SimpleJSON;
 using XUnity.AutoTranslator.Plugin.Core;
+using XUnity.AutoTranslator.Plugin.Core.Constants;
 using XUnity.AutoTranslator.Plugin.Core.Endpoints;
 using XUnity.AutoTranslator.Plugin.Core.Endpoints.Http;
 using XUnity.AutoTranslator.Plugin.Core.Extensions;
@@ -51,7 +52,7 @@ namespace BaiduTranslate
                salt,
                md5 ) );
 
-         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36" : AutoTranslatorSettings.UserAgent;
+         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? UserAgents.Chrome_Win10_Latest : AutoTranslatorSettings.UserAgent;
          request.Headers[ HttpRequestHeader.AcceptCharset ] = "UTF-8";
 
          context.Complete( request );

src/Translators/BingTranslate/BingTranslateEndpoint.cs

@@ -29,8 +29,7 @@ namespace BingTranslate
       private static readonly string HttpsServicePointTemplateUrlWithoutIG = "https://www.bing.com/ttranslate?&category=";
       private static readonly string HttpsTranslateUserSite = "https://www.bing.com/translator";
       private static readonly string RequestTemplate = "&text={0}&from={1}&to={2}";
-      private static readonly string DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36";
-      private static readonly System.Random RandomNumbers = new System.Random();
+      private static readonly Random RandomNumbers = new Random();
 
       private static readonly string[] Accepts = new string[] { "*/*" };
       private static readonly string[] AcceptLanguages = new string[] { null, "en-US,en;q=0.9", "en-US", "en" };
@@ -150,7 +149,7 @@ namespace BingTranslate
 
       private void AddHeaders( XUnityWebRequest request, bool isTranslationRequest )
       {
-         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? DefaultUserAgent : AutoTranslatorSettings.UserAgent;
+         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? UserAgents.Chrome_Win10_Latest : AutoTranslatorSettings.UserAgent;
 
          if( AcceptLanguage != null )
          {

src/Translators/GoogleTranslate/GoogleTranslateEndpoint.cs

@@ -28,7 +28,6 @@ namespace GoogleTranslate
       private static readonly string HttpsServicePointTranslateTemplateUrl = "https://translate.googleapis.com/translate_a/single?client=webapp&sl={0}&tl={1}&dt=t&tk={2}&q={3}";
       private static readonly string HttpsServicePointRomanizeTemplateUrl = "https://translate.googleapis.com/translate_a/single?client=webapp&sl={0}&tl=en&dt=rm&tk={1}&q={2}";
       private static readonly string HttpsTranslateUserSite = "https://translate.google.com";
-      private static readonly string DefaultUserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36";
       private static readonly Random RandomNumbers = new Random();
 
       private static readonly string[] Accepts = new string[] { null, "*/*", "application/json" };
@@ -189,7 +188,7 @@ namespace GoogleTranslate
 
       private void AddHeaders( XUnityWebRequest request, bool isTranslationRequest )
       {
-         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? DefaultUserAgent : AutoTranslatorSettings.UserAgent;
+         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? UserAgents.Chrome_Win10_Latest : AutoTranslatorSettings.UserAgent;
          if( AcceptLanguage != null )
          {
             request.Headers[ HttpRequestHeader.AcceptLanguage ] = AcceptLanguage;

src/Translators/GoogleTranslateLegitimate/GoogleTranslateLegitimateEndpoint.cs

@@ -20,7 +20,7 @@ namespace GoogleTranslateLegitimate
    internal class GoogleTranslateLegitimateEndpoint : HttpEndpoint
    {
       private static readonly HashSet<string> SupportedLanguages = new HashSet<string> { "af","sq","am","ar","hy","az","eu","be","bn","bs","bg","ca","ceb","zh-CN","zh-TW","co","hr","cs","da","nl","en","eo","et","fi","fr","fy","gl","ka","de","el","gu","ht","ha","haw","he","hi","hmn","hu","is","ig","id","ga","it","ja","jw","kn","kk","km","ko","ku","ky","lo","la","lv","lt","lb","mk","mg","ms","ml","mt","mi","mr","mn","my","ne","no","ny","ps","fa","pl","pt","pa","ro","ru","sm","gd","sr","st","sn","sd","si","sk","sl","so","es","su","sw","sv","tl","tg","ta","te","th","tr","uk","ur","uz","vi","cy","xh","yi","yo","zu" };
-      private static readonly string HttpsServicePointTemplateUrl = "https://translation.googleapis.com/language/translate/v2?key={0}";
+      private static readonly string HttpsServiceUrl = "https://translation.googleapis.com/language/translate/v2";
 
       private string _key;
 
@@ -44,21 +44,19 @@ namespace GoogleTranslateLegitimate
 
       public override void OnCreateRequest( IHttpRequestCreationContext context )
       {
-         var allUntranslatedText = string.Join( "\n", context.UntranslatedTexts );
-
-         var b = new StringBuilder();
-         b.Append( "{" );
-         b.Append( "\"q\":\"" ).Append( JsonHelper.Escape( allUntranslatedText ) ).Append( "\"," );
-         b.Append( "\"target\":\"" ).Append( context.DestinationLanguage ).Append( "\"," );
-         b.Append( "\"source\":\"" ).Append( context.SourceLanguage ).Append( "\"," );
-         b.Append( "\"format\":\"text\"" );
-         b.Append( "}" );
-         var data = b.ToString();
+         var urlBuilder = new StringBuilder( HttpsServiceUrl );
+         urlBuilder.Append( "?key=" ).Append( Uri.EscapeDataString( _key ) );
+         urlBuilder.Append( "&source=" ).Append( context.SourceLanguage );
+         urlBuilder.Append( "&target=" ).Append( context.DestinationLanguage );
+         for( int i = 0 ; i < context.UntranslatedTexts.Length ; i++ )
+         {
+            var untranslatedText = context.UntranslatedTexts[ i ];
+            urlBuilder.Append( "&q=" ).Append( Uri.EscapeDataString( untranslatedText ) );
+         }
 
          var request = new XUnityWebRequest(
             "POST",
-            string.Format( HttpsServicePointTemplateUrl, Uri.EscapeDataString( _key ) ),
-            data );
+            urlBuilder.ToString() );
 
          context.Complete( request );
       }
@@ -66,54 +64,18 @@ namespace GoogleTranslateLegitimate
       public override void OnExtractTranslation( IHttpTranslationExtractionContext context )
       {
          var data = context.Response.Data;
-         var obj = JSON.Parse( data );
-         var lineBuilder = new StringBuilder( data.Length );
+         var arr = JSON.Parse( data ).AsObject[ "data" ].AsObject[ "translations" ].AsArray;
 
-         foreach( JSONNode entry in obj.AsObject[ "data" ].AsObject[ "translations" ].AsArray )
+         var translatedTexts = new List<string>();
+         for( int i = 0 ; i < arr.Count ; i++ )
          {
-            var token = entry.AsObject[ "translatedText" ].ToString();
-            token = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
-
-            if( !lineBuilder.EndsWithWhitespaceOrNewline() ) lineBuilder.Append( "\n" );
-
-            lineBuilder.Append( token );
+            var obj = arr[ i ];
+            var token = obj.AsObject[ "translatedText" ].ToString();
+            var translatedText = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
+            translatedTexts.Add( translatedText );
          }
 
-         var allTranslation = lineBuilder.ToString();
-
-         if( context.UntranslatedTexts.Length == 1 )
-         {
-            context.Complete( allTranslation );
-         }
-         else
-         {
-            var translatedLines = allTranslation.Split( '\n' );
-            var translatedTexts = new List<string>();
-
-            int current = 0;
-            foreach( var untranslatedText in context.UntranslatedTexts )
-            {
-               var untranslatedLines = untranslatedText.Split( '\n' );
-               var untranslatedLinesCount = untranslatedLines.Length;
-               var translatedText = string.Empty;
-
-               for( int i = 0 ; i < untranslatedLinesCount ; i++ )
-               {
-                  if( current >= translatedLines.Length ) context.Fail( "Batch operation received incorrect number of translations." );
-
-                  var translatedLine = translatedLines[ current++ ];
-                  translatedText += translatedLine;
-
-                  if( i != untranslatedLinesCount - 1 ) translatedText += '\n';
-               }
-
-               translatedTexts.Add( translatedText );
-            }
-
-            if( current != translatedLines.Length ) context.Fail( "Batch operation received incorrect number of translations." );
-
-            context.Complete( translatedTexts.ToArray() );
-         }
+         context.Complete( translatedTexts.ToArray() );
       }
    }
 }

src/Translators/WatsonTranslate/WatsonTranslateEndpoint.cs

@@ -32,6 +32,8 @@ namespace WatsonTranslate
 
       public override string FriendlyName => "Watson Language Translator";
 
+      public override int MaxTranslationsPerRequest => 10;
+
       public override void Initialize( IInitializationContext context )
       {
          _url = context.GetOrCreateSetting( "Watson", "Url", "" );
@@ -47,15 +49,28 @@ namespace WatsonTranslate
 
       public override void OnCreateRequest( IWwwRequestCreationContext context )
       {
+         StringBuilder data = new StringBuilder();
+         data.Append( "{\"text\":[" );
+         for( int i = 0 ; i < context.UntranslatedTexts.Length ; i++ )
+         {
+            var untranslatedText = JsonHelper.Escape( context.UntranslatedTexts[ i ] );
+            data.Append( "\"" ).Append( untranslatedText ).Append( "\"" );
+
+            if( context.UntranslatedTexts.Length - 1 != i )
+            {
+               data.Append( "," );
+            }
+         }
+         data.Append( "],\"model_id\":\"" )
+            .Append( context.SourceLanguage )
+            .Append( "-" )
+            .Append( context.DestinationLanguage )
+            .Append( "\"}" );
+
          var request = new WwwRequestInfo(
             _fullUrl,
-            string.Format(
-               RequestTemplate,
-               context.SourceLanguage,
-               context.DestinationLanguage,
-               JsonHelper.Escape( context.UntranslatedText ) ) );
-
-         request.Headers[ "User-Agent" ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? "curl/7.55.1" : AutoTranslatorSettings.UserAgent;
+            data.ToString() );
+         
          request.Headers[ "Accept" ] = "application/json";
          request.Headers[ "Content-Type" ] = "application/json";
          request.Headers[ "Authorization" ] = "Basic " + Convert.ToBase64String( Encoding.ASCII.GetBytes( "apikey:" + _key ) );
@@ -66,21 +81,17 @@ namespace WatsonTranslate
       public override void OnExtractTranslation( IWwwTranslationExtractionContext context )
       {
          var data = context.ResponseData;
-         var obj = JSON.Parse( data );
-         var lineBuilder = new StringBuilder( data.Length );
+         var arr = JSON.Parse( data ).AsObject[ "translations" ].AsArray;
 
-         foreach( JSONNode entry in obj.AsObject[ "translations" ].AsArray )
+         var translatedTexts = new List<string>();
+         for( int i = 0 ; i < arr.Count ; i++ )
          {
-            var token = entry.AsObject[ "translation" ].ToString();
-            token = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
-
-            if( !lineBuilder.EndsWithWhitespaceOrNewline() ) lineBuilder.Append( "\n" );
-
-            lineBuilder.Append( token );
+            var token = arr[ i ].AsObject[ "translation" ].ToString();
+            var translatedText = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );
+            translatedTexts.Add( translatedText );
          }
-         var translated = lineBuilder.ToString();
 
-         context.Complete( translated );
+         context.Complete( translatedTexts.ToArray() );
       }
    }
 }

src/Translators/YandexTranslate/YandexTranslateEndpoint.cs

@@ -46,8 +46,7 @@ namespace YandexTranslate
                context.DestinationLanguage,
                WwwHelper.EscapeUrl( context.UntranslatedText ),
                _key ) );
-
-         request.Headers[ HttpRequestHeader.UserAgent ] = string.IsNullOrEmpty( AutoTranslatorSettings.UserAgent ) ? "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/65.0.3325.183 Safari/537.36 Vivaldi/1.96.1147.55" : AutoTranslatorSettings.UserAgent;
+         
          request.Headers[ HttpRequestHeader.Accept ] = "*/*";
          request.Headers[ HttpRequestHeader.AcceptCharset ] = "UTF-8";
 

src/XUnity.AutoTranslator.Plugin.Core/AutoTranslationPlugin.cs

@@ -673,7 +673,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
             }
          }
 
-         XuaLogger.Current.Debug( "Queued translation for: " + lookupKey );
+         XuaLogger.Current.Debug( "Queued: '" + lookupKey + "'" );
 
          ongoingJob = new TranslationJob( key );
          if( ui != null )
@@ -2134,7 +2134,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
 
                   foreach( var untranslatedText in untranslatedTexts )
                   {
-                     XuaLogger.Current.Debug( "Starting translation for: " + untranslatedText );
+                     XuaLogger.Current.Debug( "Started: '" + untranslatedText + "'" );
                   }
                   StartCoroutine(
                      _endpoint.Translate(
@@ -2164,7 +2164,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
                _ongoingJobs[ key ] = job;
 
                var untranslatedText = job.Key.GetDictionaryLookupKey();
-               XuaLogger.Current.Debug( "Starting translation for: " + untranslatedText );
+               XuaLogger.Current.Debug( "Started: " + untranslatedText );
                StartCoroutine(
                   _endpoint.Translate(
                      new[] { untranslatedText },
@@ -2205,7 +2205,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
                job.State = TranslationJobState.Succeeded;
                _ongoingJobs.Remove( job.Key.GetDictionaryLookupKey() );
 
-               XuaLogger.Current.Debug( $"Translation for '{job.Key.GetDictionaryLookupKey()}' succeded. Result: {translatedText}" );
+               XuaLogger.Current.Debug( $"Completed: '{job.Key.GetDictionaryLookupKey()}' => '{translatedText}'" );
             }
          }
          else
@@ -2252,7 +2252,7 @@ namespace XUnity.AutoTranslator.Plugin.Core
          var translatedText = translatedTextArray[ 0 ];
 
          Settings.TranslationCount++;
-         XuaLogger.Current.Debug( $"Translation for '{job.Key.GetDictionaryLookupKey()}' succeded. Result: {translatedText}" );
+         XuaLogger.Current.Debug( $"Completed: '{job.Key.GetDictionaryLookupKey()}' => '{translatedText}'" );
 
          _consecutiveErrors = 0;
 

src/XUnity.AutoTranslator.Plugin.Core/Constants/UserAgents.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace XUnity.AutoTranslator.Plugin.Core.Constants
+{
+   /// <summary>
+   /// Class defining user agents for various browsers.
+   ///
+   /// Updated 2019-02-09.
+   /// </summary>
+   public static class UserAgents
+   {
+      /// <summary>
+      /// Latest Chrome Win10 user-agent as of 2019-02-09.
+      /// </summary>
+      public static readonly string Chrome_Win10_Latest = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";
+
+      /// <summary>
+      /// Latest Chrome Win7 user-agent as of 2019-02-09.
+      /// </summary>
+      public static readonly string Chrome_Win7_Latest = "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/71.0.3578.98 Safari/537.36";
+
+      /// <summary>
+      /// Latest Firefox Win10 user-agent as of 2019-02-09.
+      /// </summary>
+      public static readonly string Firefox_Win10_Latest = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:64.0) Gecko/20100101 Firefox/64.0";
+
+      /// <summary>
+      /// Latest Edge Win10 user-agent as of 2019-02-09.
+      /// </summary>
+      public static readonly string Edge_Win10_Latest = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36 Edge/17.17134";
+   }
+}

src/XUnity.AutoTranslator.Plugin.Core/Web/XUnityWebRequest.cs

@@ -22,7 +22,19 @@ namespace XUnity.AutoTranslator.Plugin.Core.Web
          Address = new Uri( address );
          Data = data;
       }
-      
+
+      /// <summary>
+      /// Constructs a web request.
+      /// </summary>
+      /// <param name="method"></param>
+      /// <param name="address"></param>
+      public XUnityWebRequest( string method, string address )
+      {
+         Method = method;
+         Address = new Uri( address );
+         Data = string.Empty;
+      }
+
       /// <summary>
       /// Constructs a GET web request.
       /// </summary>