ExtProtocol improvement (multiline) and bug fix

Improved LEC endpoint

README.md

@@ -110,8 +110,8 @@ Language=en                      ;The language to translate into
 FromLanguage=ja                  ;The original language of the game
 
 [Files]
-Directory=Translation                                          ;Directory to search for cached translation files
-OutputFile=Translation\_AutoGeneratedTranslations.{lang}.txt   ;File to insert generated translations into
+Directory=Translation                                          ;Directory to search for cached translation files. Can use placeholder: {GameExeName}
+OutputFile=Translation\_AutoGeneratedTranslations.{lang}.txt   ;File to insert generated translations into. Can use placeholders: {GameExeName}, {lang}
 
 [TextFrameworks]
 EnableUGUI=True                  ;Enable or disable UGUI translation
@@ -143,7 +143,7 @@ TextGetterCompatibilityMode=False ;Indicates whether or not to enable "Text Gett
 GameLogTextPaths=                ;Indicates specific paths for game objects that the game uses as "log components", where it continuously appends or prepends text to. Requires expert knowledge to setup. This is a list seperated by ';'.
 
 [Texture]
-TextureDirectory=Translation\Texture ;Directory to dump textures to, and root of directories to load images from
+TextureDirectory=Translation\Texture ;Directory to dump textures to, and root of directories to load images from. Can use placeholder: {GameExeName}
 EnableTextureTranslation=False   ;Indicates whether the plugin will attempt to replace in-game images with those from the TextureDirectory directory
 EnableTextureDumping=False       ;Indicates whether the plugin will dump texture it is capable of replacing to the TextureDirectory. Has significant performance impact
 EnableTextureToggling=False      ;Indicates whether or not toggling the translation with the ALT+T hotkey will also affect textures. Not guaranteed to work for all textures. Has significant performance impact
@@ -524,7 +524,8 @@ I recommend using this class, or in case that cannot be used, falling back to th
 ### How-To
 Follow these steps:
  1. Download XUnity.AutoTranslator-Developer-{VERSION}.zip from [releases](../../releases)
- 2. Start a new project in Visual Studio 2017 or later. I recommend using the same name for your assembly/project as the "Id" you are going to use in your interface implementation. This makes it easier for users to know how to configure your translator
+ 2. Start a new project (.NET 3.5) in Visual Studio 2017 or later. I recommend using the same name for your assembly/project as the "Id" you are going to use in your interface implementation. This makes it easier for users to know how to configure your translator
+    * I recommend using the "Class Library (.NET Standard)" and simply editing the generated .csproj file to use 'net35' instead of 'netstandard2.0'. This generates much cleaner .csproj files.
  3. Add a reference to the XUnity.AutoTranslator.Plugin.Core.dll that you downloaded in step 1
  4. You do not need to directly reference the UnityEngine.dll assembly. This is good, because you do not need to worry about which version of Unity is used.
     * If you do need a reference to this assembly (because you need functionality from it) consider using an old version of it (if `UnityEngine.CoreModule.dll` exists in the Managed folder, it is not an old version!)

src/Translators/GoogleTranslate/GoogleTranslateEndpoint.cs

@@ -123,13 +123,21 @@ namespace GoogleTranslate
 
       public override void OnExtractTranslation( IHttpTranslationExtractionContext context )
       {
-         var dataIndex = context.DestinationLanguage == "romaji" ? 3 : 0;
+         var isRomaji = context.DestinationLanguage == "romaji";
+         var dataIndex = isRomaji ? 3 : 0;
 
          var data = context.Response.Data;
          var arr = JSON.Parse( data );
          var lineBuilder = new StringBuilder( data.Length );
 
-         foreach( JSONNode entry in arr.AsArray[ 0 ].AsArray )
+         arr = arr.AsArray[ 0 ];
+         if( arr.IsNull && isRomaji )
+         {
+            context.Complete( context.UntranslatedText );
+            return;
+         }
+
+         foreach( JSONNode entry in arr.AsArray )
          {
             var token = entry.AsArray[ dataIndex ].ToString();
             token = JsonHelper.Unescape( token.Substring( 1, token.Length - 2 ) );

src/Translators/Lec.ExtProtocol/Lec.ExtProtocol.csproj

@@ -6,6 +6,15 @@
     <Platforms>AnyCPU;x86</Platforms>
   </PropertyGroup>
 
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|x86'">
+    <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x86'">
+    <Optimize>false</Optimize>
+    <AllowUnsafeBlocks>false</AllowUnsafeBlocks>
+  </PropertyGroup>
+
    <Target Name="PostBuild" AfterTargets="PostBuildEvent">
       <Exec Command="if $(ConfigurationName) == Release (&#xD;&#xA;   XCOPY /Y /I &quot;$(TargetDir)$(TargetName)$(TargetExt)&quot; &quot;$(SolutionDir)dist\Translators\&quot;&#xD;&#xA;)" />
    </Target>

src/Translators/Lec.ExtProtocol/LecTranslationLibrary.cs

@@ -16,7 +16,7 @@ namespace Lec.ExtProtocol
       [UnmanagedFunctionPointer( CallingConvention.Cdecl )]
       private delegate int eg_end();
       [UnmanagedFunctionPointer( CallingConvention.Cdecl )]
-      private delegate int eg_translate_multi( int i, IntPtr in_str, int out_size, StringBuilder out_str );
+      private delegate int eg_translate_multi( int i, IntPtr in_str, int out_size, IntPtr out_str );
 
       private eg_end _end;
       private eg_translate_multi _translate;
@@ -73,8 +73,9 @@ namespace Lec.ExtProtocol
       {
          toTranslate = PreprocessString( toTranslate );
          var size = toTranslate.Length * 3;
-         var builder = new StringBuilder();
          int translatedSize;
+         IntPtr ptr = IntPtr.Zero;
+
          // we can't know the output size, so just try until we have a big enough buffer
          do
          {
@@ -85,16 +86,30 @@ namespace Lec.ExtProtocol
                return null;
             }
 
-            builder.Capacity = size;
+            if( ptr != IntPtr.Zero )
+            {
+               Marshal.FreeHGlobal( ptr );
+            }
+            ptr = Marshal.AllocHGlobal( size );
+
             var str = ConvertStringToNative( toTranslate, JapaneseCodepage );
-            translatedSize = _translate( 0, str, size, builder );
+            translatedSize = _translate( 0, str, size, ptr );
+
+
             Marshal.FreeHGlobal( str );
 
          } while( translatedSize > size );
 
-         return builder.ToString();
+         var result = ConvertNativeToString( ptr, JapaneseCodepage );
+
+         if( ptr != IntPtr.Zero )
+         {
+            Marshal.FreeHGlobal( ptr );
+         }
+
+         return result;
       }
-      
+
       protected override bool OnInitialize( string libraryPath )
       {
          try

src/Translators/Lec.ExtProtocol/Program.cs

@@ -41,12 +41,18 @@ namespace Lec.ExtProtocol
                      var message = ExtProtocolConvert.Decode( receivedPayload ) as TranslationRequest;
                      if( message == null ) return;
 
-                     var translatedLine = translator.Translate( message.UntranslatedText );
+                     var translatedTexts = new string[ message.UntranslatedTexts.Length ];
+                     for( int i = 0 ; i < message.UntranslatedTexts.Length ; i++ )
+                     {
+                        var untranslatedText = message.UntranslatedTexts[ i ];
+                        var translatedText = translator.Translate( untranslatedText );
+                        translatedTexts[ i ] = translatedText;
+                     }
 
                      var response = new TranslationResponse
                      {
                         Id = message.Id,
-                        TranslatedText = translatedLine
+                        TranslatedTexts = translatedTexts
                      };
 
                      var translatedPayload = ExtProtocolConvert.Encode( response );

src/Translators/Lec.ExtProtocol/UnmanagedTranslationLibrary.cs

@@ -39,13 +39,13 @@ namespace Lec.ExtProtocol
          return nativeUtf8;
       }
 
-      public static string ConvertNativeToString( IntPtr nativeUtf8 )
+      public static string ConvertNativeToString( IntPtr nativeUtf8, int codepage )
       {
          int len = 0;
          while( Marshal.ReadByte( nativeUtf8, len ) != 0 ) ++len;
          byte[] buffer = new byte[ len ];
          Marshal.Copy( nativeUtf8, buffer, 0, buffer.Length );
-         return Encoding.UTF8.GetString( buffer );
+         return Encoding.GetEncoding( codepage ).GetString( buffer );
       }
 
       protected virtual void Dispose( bool disposing )

src/Translators/LecPowerTranslator15/LecPowerTranslator15Endpoint.cs

@@ -15,7 +15,9 @@ namespace LecPowerTranslator15
 
       public override string FriendlyName => "LEC Power Translator 15";
 
-      public override int MaxConcurrency => 15;
+      public override int MaxConcurrency => 1;
+
+      public override int MaxTranslationsPerRequest => 50;
 
       public override void Initialize( IInitializationContext context )
       {

src/XUnity.AutoTranslator.Plugin.Core/Endpoints/ExtProtocol/ExtProtocolEndpoint.cs

@@ -47,7 +47,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
       /// <summary>
       /// Gets the maximum number of translations that can be served per translation request.
       /// </summary>
-      public int MaxTranslationsPerRequest => 1;
+      public virtual int MaxTranslationsPerRequest => 1;
 
       /// <summary>
       /// Gets the path to the executable that should be communicated with.
@@ -184,7 +184,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
                Id = id,
                SourceLanguage = context.SourceLanguage,
                DestinationLanguage = context.DestinationLanguage,
-               UntranslatedText = context.UntranslatedText
+               UntranslatedTexts = context.UntranslatedTexts
             };
             var payload = ExtProtocolConvert.Encode( request );
 
@@ -201,7 +201,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
 
          if( !result.Succeeded ) context.Fail( "Error occurred while retrieving translation. " + result.Error );
 
-         context.Complete( result.Result );
+         context.Complete( result.Results );
       }
 
       private void HandleProtocolMessageResponse( ProtocolMessage message )
@@ -225,7 +225,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
          {
             if( _transactionHandles.TryGetValue( message.Id, out var result ) )
             {
-               result.SetCompleted( message.TranslatedText, null );
+               result.SetCompleted( message.TranslatedTexts, null );
                _transactionHandles.Remove( message.Id );
             }
          }

src/XUnity.AutoTranslator.Plugin.Core/Endpoints/ExtProtocol/ProtocolTransactionHandle.cs

@@ -10,10 +10,10 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
          StartTime = Time.realtimeSinceStartup;
       }
 
-      public void SetCompleted( string result, string error )
+      public void SetCompleted( string[] translatedTexts, string error )
       {
          IsCompleted = true;
-         Result = result;
+         Results = translatedTexts;
          Error = error;
       }
 
@@ -21,7 +21,7 @@ namespace XUnity.AutoTranslator.Plugin.Core.Endpoints.ExtProtocol
 
       public float StartTime { get; set; }
 
-      public string Result { get; set; }
+      public string[] Results { get; set; }
 
       public string Error { get; set; }
 

src/XUnity.AutoTranslator.Plugin.ExtProtocol/ExtProtocolConvert.cs

@@ -43,7 +43,7 @@ namespace XUnity.AutoTranslator.Plugin.ExtProtocol
          var id = TypeToId[ message.GetType() ];
          writer.WriteLine( id );
          message.Encode( writer );
-         return Convert.ToBase64String( Encoding.UTF8.GetBytes( writer.ToString() ) );
+         return Convert.ToBase64String( Encoding.UTF8.GetBytes( writer.ToString() ), Base64FormattingOptions.None );
       }
 
       /// <summary>

src/XUnity.AutoTranslator.Plugin.ExtProtocol/TranslationRequest.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Globalization;
 using System.IO;
+using System.Text;
 
 namespace XUnity.AutoTranslator.Plugin.ExtProtocol
 {
@@ -27,14 +29,22 @@ namespace XUnity.AutoTranslator.Plugin.ExtProtocol
       /// <summary>
       /// Gets or sets the untranslated text.
       /// </summary>
-      public string UntranslatedText { get; set; }
+      public string[] UntranslatedTexts { get; set; }
 
       internal override void Decode( TextReader reader )
       {
          Id = new Guid( reader.ReadLine() );
          SourceLanguage = reader.ReadLine();
          DestinationLanguage = reader.ReadLine();
-         UntranslatedText = reader.ReadToEnd();
+         var count = int.Parse( reader.ReadLine(), CultureInfo.InvariantCulture );
+         var untranslatedTexts = new string[ count ];
+         for( int i = 0 ; i < count ; i++ )
+         {
+            var encodedUntranslatedText = reader.ReadLine();
+            var untranslatedText = Encoding.UTF8.GetString( Convert.FromBase64String( encodedUntranslatedText ) );
+            untranslatedTexts[ i ] = untranslatedText;
+         }
+         UntranslatedTexts = untranslatedTexts;
       }
 
       internal override void Encode( TextWriter writer )
@@ -42,7 +52,12 @@ namespace XUnity.AutoTranslator.Plugin.ExtProtocol
          writer.WriteLine( Id.ToString() );
          writer.WriteLine( SourceLanguage );
          writer.WriteLine( DestinationLanguage );
-         writer.Write( UntranslatedText );
+         writer.WriteLine( UntranslatedTexts.Length.ToString( CultureInfo.InvariantCulture ) );
+         foreach( var untranslatedText in UntranslatedTexts )
+         {
+            var encodedUntranslatedText = Convert.ToBase64String( Encoding.UTF8.GetBytes( untranslatedText ), Base64FormattingOptions.None );
+            writer.WriteLine( encodedUntranslatedText );
+         }
       }
    }
 }

src/XUnity.AutoTranslator.Plugin.ExtProtocol/TranslationResponse.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Globalization;
 using System.IO;
+using System.Text;
 
 namespace XUnity.AutoTranslator.Plugin.ExtProtocol
 {
@@ -16,18 +18,31 @@ namespace XUnity.AutoTranslator.Plugin.ExtProtocol
       /// <summary>
       /// Gets or sets the translated text.
       /// </summary>
-      public string TranslatedText { get; set; }
+      public string[] TranslatedTexts { get; set; }
 
       internal override void Decode( TextReader reader )
       {
          Id = new Guid( reader.ReadLine() );
-         TranslatedText = reader.ReadToEnd();
+         var count = int.Parse( reader.ReadLine(), CultureInfo.InvariantCulture );
+         var translatedTexts = new string[ count ];
+         for( int i = 0 ; i < count ; i++ )
+         {
+            var encodedTranslatedText = reader.ReadLine();
+            var translatedText = Encoding.UTF8.GetString( Convert.FromBase64String( encodedTranslatedText ) );
+            translatedTexts[ i ] = translatedText;
+         }
+         TranslatedTexts = translatedTexts;
       }
 
       internal override void Encode( TextWriter writer )
       {
          writer.WriteLine( Id.ToString() );
-         writer.Write( TranslatedText );
+         writer.WriteLine( TranslatedTexts.Length.ToString( CultureInfo.InvariantCulture ) );
+         foreach( var translatedText in TranslatedTexts )
+         {
+            var encodedTranslatedText = Convert.ToBase64String( Encoding.UTF8.GetBytes( translatedText ), Base64FormattingOptions.None );
+            writer.WriteLine( encodedTranslatedText );
+         }
       }
    }
 }