From 74760a76614345b3c1c172e1ffd4108e047473ca Mon Sep 17 00:00:00 2001 From: Ari Vuollet Date: Wed, 21 Jun 2023 13:14:27 +0300 Subject: [PATCH 1/2] Add support for writing UTF-8 files in FileBase::WriteAllText --- Source/Engine/Core/Encoding.h | 2 +- Source/Engine/Platform/Base/FileBase.cpp | 39 ++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/Source/Engine/Core/Encoding.h b/Source/Engine/Core/Encoding.h index 6ac716ba8..3e74144e2 100644 --- a/Source/Engine/Core/Encoding.h +++ b/Source/Engine/Core/Encoding.h @@ -4,4 +4,4 @@ #include "Enums.h" -DECLARE_ENUM_3(Encoding, ANSI, Unicode, UnicodeBigEndian); +DECLARE_ENUM_4(Encoding, ANSI, Unicode, UnicodeBigEndian, UTF8); diff --git a/Source/Engine/Platform/Base/FileBase.cpp b/Source/Engine/Platform/Base/FileBase.cpp index f45685c7f..4794e2452 100644 --- a/Source/Engine/Platform/Base/FileBase.cpp +++ b/Source/Engine/Platform/Base/FileBase.cpp @@ -309,6 +309,45 @@ bool FileBase::WriteAllText(const StringView& path, const Char* data, int32 leng return WriteAllBytes(path, tmp.Get(), tmp.Count()); } + case Encoding::UTF8: + { + Array tmp; + tmp.SetCapacity(length); + + for (int i=0; i> 6)); + tmp.Add(0x80 | (c & 0x3F)); + } + else if (c < 0xD800 || c >= 0xE000) + { + tmp.Add(0xE0 | (c >> 12)); + tmp.Add(0x80 | ((c >> 6) & 0x3F)); + tmp.Add(0x80 | (c & 0x3F)); + } + else // Surrogate pair + { + ++i; + if (i >= length) + return true; + + uint32 p = 0x10000 + (((c & 0x3FF) << 10) | (data[i] & 0x3FF)); + tmp.Add(0xF0 | (p >> 18)); + tmp.Add(0x80 | ((p >> 12) & 0x3F)); + tmp.Add(0x80 | ((p >> 6) & 0x3F)); + tmp.Add(0x80 | (p & 0x3F)); + } + } + + return WriteAllBytes(path, tmp.Get(), tmp.Count()); + } default: return true; } From 83485d2f528cfeb52c12ef7193a22d0dbcffa3c8 Mon Sep 17 00:00:00 2001 From: Ari Vuollet Date: Wed, 21 Jun 2023 13:16:49 +0300 Subject: [PATCH 2/2] Write generated build configuration files in UTF-8 Git seems to have issues with handling UTF-16 encoded files, so it would be more appropriate to write the generated files with more common UTF-8 encoding instead. --- Source/Editor/Editor.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Source/Editor/Editor.cpp b/Source/Editor/Editor.cpp index 8a92d11f4..527bfeb6d 100644 --- a/Source/Editor/Editor.cpp +++ b/Source/Editor/Editor.cpp @@ -184,7 +184,7 @@ bool Editor::CheckProjectUpgrade() " BuildNativeCode = false;\n" " }}\n" "}}\n" - ), codeName), Encoding::Unicode); + ), codeName), Encoding::UTF8); if (useEditorModule) { File::WriteAllText(gameEditorModuleFolder / String::Format(TEXT("{0}Editor.Build.cs"), codeName), String::Format(TEXT( @@ -211,7 +211,7 @@ bool Editor::CheckProjectUpgrade() " BuildNativeCode = false;\n" " }}\n" "}}\n" - ), codeName), Encoding::Unicode); + ), codeName), Encoding::UTF8); } // Generate target files @@ -229,7 +229,7 @@ bool Editor::CheckProjectUpgrade() " Modules.Add(\"{0}\");\n" " }}\n" "}}\n" - ), codeName), Encoding::Unicode); + ), codeName), Encoding::UTF8); const String editorTargetGameEditorModule = useEditorModule ? String::Format(TEXT(" Modules.Add(\"{0}Editor\");\n"), codeName) : String::Empty; File::WriteAllText(sourceFolder / String::Format(TEXT("{0}EditorTarget.Build.cs"), codeName), String::Format(TEXT( "using Flax.Build;\n" @@ -246,7 +246,7 @@ bool Editor::CheckProjectUpgrade() "{1}" " }}\n" "}}\n" - ), codeName, editorTargetGameEditorModule), Encoding::Unicode); + ), codeName, editorTargetGameEditorModule), Encoding::UTF8); // Generate new project file Project->ProjectPath = root / String::Format(TEXT("{0}.flaxproj"), codeName); @@ -454,7 +454,7 @@ int32 Editor::LoadProduct() " // Reference the modules for game\n" " Modules.Add(\"Game\");\n" " }\n" - "}\n"), Encoding::Unicode); + "}\n"), Encoding::UTF8); failed |= File::WriteAllText(projectPath / TEXT("Source/GameEditorTarget.Build.cs"),TEXT( "using Flax.Build;\n" "\n" @@ -468,7 +468,7 @@ int32 Editor::LoadProduct() " // Reference the modules for editor\n" " Modules.Add(\"Game\");\n" " }\n" - "}\n"), Encoding::Unicode); + "}\n"), Encoding::UTF8); failed |= File::WriteAllText(projectPath / TEXT("Source/Game/Game.Build.cs"),TEXT( "using Flax.Build;\n" "using Flax.Build.NativeCpp;\n" @@ -496,7 +496,7 @@ int32 Editor::LoadProduct() " // To add C++ define use: options.PublicDefinitions.Add(\"COMPILE_WITH_FLAX\");\n" " // To learn more see scripting documentation.\n" " }\n" - "}\n"), Encoding::Unicode); + "}\n"), Encoding::UTF8); if (failed) return 12; }