Merge pull request #124 from lassoan/utf8-encoding-for-windows

jcfr · web-flow · commit 1788b378ed2e · 2020-02-20T11:35:25.000-08:00
ENH: Allow applications use UTF-8 code page
diff --git a/CMake/SEMFunctionAddExecutable.cmake b/CMake/SEMFunctionAddExecutable.cmake
@@ -0,0 +1,22 @@
+#! Usage:
+#! \code
+#! sem_add_executable(args)
+#! \endcode
+#!
+#! This macro adds an executable that uses UTF-8 code page
+#! if SlicerExecutionModel_USE_UTF8 is enabled.
+#!
+#! Linux and Mac OSX already uses this code page by default but on Windows
+#! it has to be set explicitly, by using a manifest file.
+#!
+#! \ingroup CMakeUtilities
+
+function(sem_add_executable)
+
+if(SlicerExecutionModel_USE_UTF8 AND WIN32)
+  add_executable(${ARGN} ${SlicerExecutionModel_CMAKE_DIR}/WindowsApplicationUseUtf8.manifest)
+else()
+  add_executable(${ARGN})
+endif()
+
+endfunction()
diff --git a/CMake/SEMMacroBuildCLI.cmake b/CMake/SEMMacroBuildCLI.cmake
@@ -156,7 +156,7 @@ macro(SEMMacroBuildCLI)
     set_target_properties(${CLP}Lib PROPERTIES LINK_FLAGS ${_cli_library_link_flags})
   endif()
 
-  add_executable(${CLP} ${LOCAL_SEM_CLI_LIBRARY_WRAPPER_CXX})
+  sem_add_executable(${CLP} ${LOCAL_SEM_CLI_LIBRARY_WRAPPER_CXX})
   set_target_properties(${CLP} PROPERTIES COMPILE_FLAGS "${cli_executable_compile_flags}")
   set(cli_executable_libraries ${CLP}Lib)
   if(DEFINED SlicerExecutionModel_EXTRA_EXECUTABLE_TARGET_LIBRARIES)
diff --git a/CMake/WindowsApplicationUseUtf8.manifest b/CMake/WindowsApplicationUseUtf8.manifest
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v3">
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -73,6 +73,14 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     "MinSizeRel" "RelWithDebInfo")
 endif()
 
+#-----------------------------------------------------------------------------
+# ParameterSerializer support for ModuleDescriptionParser and GenerateCLP subprojects
+#-----------------------------------------------------------------------------
+option(SlicerExecutionModel_USE_UTF8 "Make applications use UTF-8 as code page." OFF)
+set(SlicerExecutionModel_CMAKE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/CMake)
+set(GenerateCLP_USE_UTF8 ${SlicerExecutionModel_USE_UTF8})
+include(SEMFunctionAddExecutable)
+
 #-----------------------------------------------------------------------------
 # Output directories associated with ModuleDescriptionParser and GenerateCLP subprojects
 #-----------------------------------------------------------------------------
@@ -229,8 +237,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/UseSlicerExecutionModel.cmake.in
 set(allscripts
   CMake/FindJsonCpp.cmake
   CMake/SEMCommandLineLibraryWrapper.cxx
+  CMake/SEMFunctionAddExecutable.cmake
   CMake/SEMMacroBuildCLI.cmake
   CMake/SEMToolTestName.cxx.in
+  CMake/WindowsApplicationUseUtf8.manifest
   )
 foreach(SCRIPT ${allscripts})
   get_filename_component(_fileName ${SCRIPT} NAME)
diff --git a/GenerateCLP/CMakeLists.txt b/GenerateCLP/CMakeLists.txt
@@ -105,7 +105,7 @@ set(executable_name GenerateCLP)
 set(GENERATECLP_SOURCE
   GenerateCLP.cxx
   )
-add_executable(${executable_name} ${GENERATECLP_SOURCE})
+sem_add_executable(${executable_name} ${GENERATECLP_SOURCE})
 list(APPEND targets_to_export ${executable_name})
 
 target_link_libraries(${executable_name}
@@ -215,7 +215,7 @@ add_custom_command(TARGET GenerateCLP POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E touch ${GenerateCLPLauncher_SOURCE}
   COMMENT "Force GenerateCLPLauncher to rebuild after GenerateCLP was modified"
   )
-add_executable(GenerateCLPLauncher
+sem_add_executable(GenerateCLPLauncher
   ${GenerateCLPLauncher_SOURCE}
   )
 list(APPEND targets_to_export GenerateCLPLauncher)
diff --git a/GenerateCLP/GenerateCLPConfig.cmake.in b/GenerateCLP/GenerateCLPConfig.cmake.in
@@ -7,6 +7,8 @@ set(GenerateCLP_CMAKE_DIR "@GenerateCLP_CMAKE_DIR_CONFIG@")
 set(GenerateCLP_USE_FILE "@GenerateCLP_USE_FILE_CONFIG@")
 set(GenerateCLP_USE_JSONCPP "@GenerateCLP_USE_JSONCPP@")
 set(GenerateCLP_USE_SERIALIZER "@GenerateCLP_USE_SERIALIZER@")
+set(SlicerExecutionModel_USE_UTF8 "@SlicerExecutionModel_USE_UTF8@")
+set(SlicerExecutionModel_CMAKE_DIR "@SlicerExecutionModel_CMAKE_DIR@")
 set(TCLAP_DIR "@TCLAP_DIR@")
 set(ModuleDescriptionParser_DIR "@ModuleDescriptionParser_DIR@")
 set(ITK_DIR "@ITK_DIR_CONFIG@")
@@ -29,3 +31,5 @@ find_program(GENERATECLP_EXE
 if(NOT TARGET GenerateCLP AND EXISTS "@CMAKE_CURRENT_BINARY_DIR@/GenerateCLPTargets.cmake")
   include("@CMAKE_CURRENT_BINARY_DIR@/GenerateCLPTargets.cmake")
 endif()
+
+include("${SlicerExecutionModel_CMAKE_DIR}/SEMFunctionAddExecutable.cmake")
diff --git a/GenerateCLP/GenerateCLPConfig.h.in b/GenerateCLP/GenerateCLPConfig.h.in
@@ -29,6 +29,7 @@
 #cmakedefine GENERATECLP_USE_MD5
 #cmakedefine GenerateCLP_USE_JSONCPP
 #cmakedefine GenerateCLP_USE_SERIALIZER
+#cmakedefine SlicerExecutionModel_USE_UTF8
 
 #endif
 
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTest-Configure.cmake b/GenerateCLP/Testing/CMake/GenerateCLPTest-Configure.cmake
@@ -30,6 +30,7 @@ set(command ${CMAKE_COMMAND}
   -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD}
   -DGenerateCLP_DIR:PATH=${GenerateCLP_BINARY_DIR}
   -DGenerateCLP_USE_JSONCPP:BOOL=${GenerateCLP_USE_JSONCPP}
+  -DSlicerExecutionModel_USE_UTF8:BOOL=${SlicerExecutionModel_USE_UTF8}
   -DJsonCpp_CMAKE_MODULE_PATH:PATH=${JsonCpp_CMAKE_MODULE_PATH}
   -G ${generateclp_cmake_generator} ${TEST_SOURCE_DIR})
 execute_process(
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTestMacros.cmake b/GenerateCLP/Testing/CMake/GenerateCLPTestMacros.cmake
@@ -71,7 +71,7 @@ macro(GenerateCLP_TEST_PROJECT)
   #-----------------------------------------------------------------------------
   set(${PROJECT_NAME}_SOURCE ${PROJECT_NAME}.cxx)
   GENERATECLP(${PROJECT_NAME}_SOURCE ${PROJECT_NAME}.xml)
-  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCE})
+  sem_add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCE})
   if(_additional_link_libraries)
     target_link_libraries(${PROJECT_NAME} ${_additional_link_libraries})
   endif()
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTestPrerequisites.cmake.in b/GenerateCLP/Testing/CMake/GenerateCLPTestPrerequisites.cmake.in
@@ -13,6 +13,9 @@ set(JsonCpp_LIBRARY "@JsonCpp_LIBRARY@")
 
 set(JsonCpp_CMAKE_MODULE_PATH "@SlicerExecutionModel_SOURCE_DIR@/CMake")
 
+set(SlicerExecutionModel_USE_UTF8 @SlicerExecutionModel_USE_UTF8@)
+set(SlicerExecutionModel_CMAKE_DIR "@SlicerExecutionModel_CMAKE_DIR@")
+
 # --------------------------------------------------------------------------
 # Sanity checks
 
diff --git a/GenerateCLP/Testing/CMakeLists.txt b/GenerateCLP/Testing/CMakeLists.txt
@@ -49,3 +49,10 @@ set(_previous_test "NODEPENDS")
 generateclp_add_test(CLPTestMultiple Configure)
 generateclp_add_test(CLPTestMultiple Build)
 generateclp_add_test(CLPTestMultiple Test)
+
+if(SlicerExecutionModel_USE_UTF8)
+  set(_previous_test "NODEPENDS")
+  generateclp_add_test(Utf8Example Configure)
+  generateclp_add_test(Utf8Example Build)
+  generateclp_add_test(Utf8Example Test)
+endif()
diff --git a/GenerateCLP/Testing/Utf8Example/CMakeLists.txt b/GenerateCLP/Testing/Utf8Example/CMakeLists.txt
@@ -0,0 +1,16 @@
+#-----------------------------------------------------------------------------
+cmake_minimum_required(VERSION 3.13.4...3.14.2)
+cmake_policy(VERSION 3.13.4...3.14.2)
+#-----------------------------------------------------------------------------
+# Configure & Build
+#-----------------------------------------------------------------------------
+include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/GenerateCLPTestMacros.cmake)
+GenerateCLP_TEST_PROJECT(
+  NAME Utf8Example1
+)
+
+#-----------------------------------------------------------------------------
+# Test
+#-----------------------------------------------------------------------------
+add_test(NAME SlicerCLPExample1 COMMAND $<TARGET_FILE:${PROJECT_NAME}> --help)
+add_test(NAME SlicerCLPExample2 COMMAND $<TARGET_FILE:${PROJECT_NAME}>)
diff --git a/GenerateCLP/Testing/Utf8Example/Utf8Example1.cxx b/GenerateCLP/Testing/Utf8Example/Utf8Example1.cxx
@@ -0,0 +1,135 @@
+#ifdef _WIN32
+#include "Windows.h"
+#endif
+
+#include "Utf8Example1CLP.h"
+
+#include <iostream>
+#include <fstream>
+
+#ifdef _WIN32
+
+// Helper function to get Windows version
+typedef LONG NTSTATUS, * PNTSTATUS;
+#define STATUS_SUCCESS (0x00000000)
+typedef NTSTATUS(WINAPI* RtlGetVersionPtr)(PRTL_OSVERSIONINFOW);
+RTL_OSVERSIONINFOW GetRealOSVersion()
+{
+  HMODULE hMod = ::GetModuleHandleW(L"ntdll.dll");
+  if (hMod)
+  {
+    RtlGetVersionPtr fxPtr = (RtlGetVersionPtr)::GetProcAddress(hMod, "RtlGetVersion");
+    if (fxPtr != nullptr)
+    {
+      RTL_OSVERSIONINFOW rovi = { 0 };
+      rovi.dwOSVersionInfoSize = sizeof(rovi);
+      if (STATUS_SUCCESS == fxPtr(&rovi))
+      {
+        return rovi;
+      }
+    }
+  }
+  RTL_OSVERSIONINFOW rovi = { 0 };
+  return rovi;
+}
+
+#endif
+
+int main (int argc, char *argv[])
+{
+  PARSE_ARGS;
+
+  std::cout << "inputStringEnum: " << inputStringEnum << std::endl;
+  std::cout << "inputFile: " << inputFile << std::endl;
+  std::cout << "outputFile: " << outputFile << std::endl;
+
+  // Test file writing
+  std::ofstream outputFileStream;
+  outputFileStream.open(outputFile);
+  if (!outputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for writing: " << outputFile << std::endl;
+    return 1;
+  }
+  outputFileStream << inputStringEnum << std::endl;
+  outputFileStream.close();
+
+  // Test file reading
+  std::ifstream inputFileStream;
+  inputFileStream.open(outputFile);
+  if (!inputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for reading: " << inputFile << std::endl;
+  }
+  std::string writtenReadStringEnum;
+  getline(inputFileStream, writtenReadStringEnum);
+  inputFileStream.close();
+  std::cout << "written and read inputStringEnum: " << writtenReadStringEnum << std::endl;
+
+  // Test output string
+  // Write out the return parameters in "name = value" form
+  std::ofstream rts;
+  rts.open(returnParameterFile.c_str());
+  rts << "outputString = " << inputStringEnum << std::endl;
+  rts.close();
+
+#ifdef _WIN32
+  
+  // Check current windows version before proceeding
+  RTL_OSVERSIONINFOW rovi = GetRealOSVersion();
+  std::cout << "Windows version: " << rovi.dwMajorVersion << "." << rovi.dwMinorVersion << " build " << rovi.dwBuildNumber << std::endl;
+  if (rovi.dwBuildNumber < 18362)
+  {
+    std::cout << "This Windows version does not support UTF-8 as active code page in application (minimum build number is 18362). Further testing is skipped." << std::endl;
+    return 0;
+  }
+
+  // Check that active code page is UTF-8 on Windows
+  UINT activeCodePage = GetACP();
+  std::cout << "Active code page: " << activeCodePage << std::endl;
+  if (activeCodePage != CP_UTF8)
+  {
+    std::cerr << "Error: active code page is " << activeCodePage << ", expected " << CP_UTF8 << " (UTF-8)" << std::endl;
+    return 1;
+  }
+#endif
+
+  // Check that we can create a file wit utf8 filename using standard file API
+  std::string filenameUtf8 = u8"alpha(\u03b1).txt";
+  outputFileStream.open(filenameUtf8);
+  if (!outputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for writing: " << outputFile << std::endl;
+    return 1;
+  }
+  outputFileStream << inputStringEnum << std::endl;
+  outputFileStream.close();
+
+  // Test if file is created successfully
+  // We use special wide character API on Windows to ensure we properly check for the existence of the file
+  // regardless of active code page.
+#if _WIN32
+  std::wstring filenameW;
+  filenameW += L"alpha(";
+  filenameW.push_back((wchar_t)(0x03B1));
+  filenameW += L").txt";
+  FILE* tmp = _wfopen(filenameW.c_str(), L"r");
+#else
+  FILE* tmp = fopen(filenameUtf8.c_str(), "r");
+#endif
+  if (!tmp)
+  {
+    std::cerr << "Expected file does not exist: " << filenameUtf8 << std::endl;
+    return 1;
+  }
+  fclose(tmp);
+
+  // Delete the temporary file
+#if _WIN32
+  _wunlink(filenameW.c_str());
+#else
+  unlink(utf8_str.c_str());
+#endif
+
+  return 0;
+}
diff --git a/GenerateCLP/Testing/Utf8Example/Utf8Example1.xml b/GenerateCLP/Testing/Utf8Example/Utf8Example1.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8"?>
+<executable>
+  <category>utf8</category>
+  <title>Test special characters oØo</title>
+  <description>This test checks if special characters can be used in SEM</description>
+  <version>0.1.0.$Revision$(alpha)</version>
+  <documentation-url></documentation-url>
+  <license></license>
+  <contributor>András Lassó</contributor>
+
+  <parameters>
+    <label>Section låbel with an ao</label>
+    <description>Section dæscription with an ae</description>
+    <string-enumeration>
+      <name>inputStringEnum</name>
+      <longflag>--inputStringEnum</longflag>
+      <description>Enumerated strings test</description>
+      <label>Enumerated input string</label>
+      <default>Hungarian: árvíztűrő tükörfúrógép</default>
+      <element>Hungarian: árvíztűrő tükörfúrógép</element>
+      <element>Nordic: æ (ae) Æ (AE) ä (a:) Ä (A:) ø (o|) Ø (O|) ö (o:) Ö (O:) å (ao) Å (Ao)</element>
+      <element>Japanese: 日本</element>
+    </string-enumeration>
+    <file>
+      <name>inputFile</name>
+      <longflag>--inputFile</longflag>
+      <description>Generic input file</description>
+      <label>Input file</label>
+      <default><![CDATA[input-test-日本-árvíztűrő-tükörfúrógép.txt]]></default>
+    </file>
+    <file>
+      <name>outputFile</name>
+      <longflag>--outputFile</longflag>
+      <description>Generic output file</description>
+      <label>Output file</label>
+      <default>output-test-日本-árvíztűrő-tükörfúrógép.txt</default>
+    </file>
+    <string>
+      <name>outputString</name>
+      <label>Output string</label>
+      <channel>output</channel>
+      <default>árvíztűrő 日本</default>
+      <description><![CDATA[An example of a string return type]]></description>
+    </string>
+  </parameters>
+</executable>
diff --git a/ModuleDescriptionParser/Testing/CMakeLists.txt b/ModuleDescriptionParser/Testing/CMakeLists.txt
@@ -6,7 +6,7 @@ create_test_sourcelist(Tests ${KIT}CxxTests.cxx
   ModuleDescriptionTest.cxx
   )
 
-add_executable(${KIT}CxxTests
+sem_add_executable(${KIT}CxxTests
   ModuleDescriptionTestingMacros.h
   ModuleDescriptionTestingUtilities.cxx
   ModuleDescriptionTestingUtilities.h
@@ -18,7 +18,7 @@ target_link_libraries(${KIT}CxxTests ${KIT})
 # Build Parser1Test executable
 # --------------------------------------------------------------------------
 set(PARSER1TEST_SOURCE Parser1Test.cxx )
-add_executable(Parser1Test ${PARSER1TEST_SOURCE})
+sem_add_executable(Parser1Test ${PARSER1TEST_SOURCE})
 target_link_libraries(Parser1Test ModuleDescriptionParser )
 
 # --------------------------------------------------------------------------
diff --git a/ModuleDescriptionParser/Testing/TestData/parameter-file-with-pointfile-slicer-issue2979.params b/ModuleDescriptionParser/Testing/TestData/parameter-file-with-pointfile-slicer-issue2979.params
@@ -1,2 +1,2 @@
-Input Fiducial File = input.fcsv
-Output Fiducial File = output.fcsv
+Input Fiducial File = input.fcsv
+Output Fiducial File = output.fcsv
diff --git a/SlicerExecutionModelConfig.cmake.in b/SlicerExecutionModelConfig.cmake.in
@@ -53,6 +53,10 @@ if(SlicerExecutionModel_USE_SERIALIZER AND NOT SlicerExecutionModel_USE_JSONCPP)
   Make sure SlicerExecutionModel_USE_JSONCPP is ON.")
 endif()
 
+# SlicerExecutionModel_USE_UTF8 is optional and it must be turned on at
+# build time to make sure active code page is consistent in all executables.
+set(SlicerExecutionModel_USE_UTF8 @SlicerExecutionModel_USE_UTF8@)
+
 # Set SlicerExecutionModel variables
 set(SlicerExecutionModel_INCLUDE_DIRS
   "@SlicerExecutionModel_INCLUDE_DIRS_CONFIG@"
diff --git a/UseSlicerExecutionModel.cmake.in b/UseSlicerExecutionModel.cmake.in
@@ -3,4 +3,4 @@ include_directories(BEFORE ${SlicerExecutionModel_INCLUDE_DIRS})
 set(CMAKE_MODULE_PATH ${SlicerExecutionModel_CMAKE_DIR} ${CMAKE_MODULE_PATH})
 
 include(SEMMacroBuildCLI)
-
+include(SEMFunctionAddExecutable)

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,7 @@ set(executable_name GenerateCLP)`
`105`	`105`	`set(GENERATECLP_SOURCE`
`106`	`106`	`GenerateCLP.cxx`
`107`	`107`	`)`
`108`		`-add_executable(${executable_name} ${GENERATECLP_SOURCE})`
	`108`	`+sem_add_executable(${executable_name} ${GENERATECLP_SOURCE})`
`109`	`109`	`list(APPEND targets_to_export ${executable_name})`
`110`	`110`
`111`	`111`	`target_link_libraries(${executable_name}`
`@@ -215,7 +215,7 @@ add_custom_command(TARGET GenerateCLP POST_BUILD`
`215`	`215`	`COMMAND ${CMAKE_COMMAND} -E touch ${GenerateCLPLauncher_SOURCE}`
`216`	`216`	`COMMENT "Force GenerateCLPLauncher to rebuild after GenerateCLP was modified"`
`217`	`217`	`)`
`218`		`-add_executable(GenerateCLPLauncher`
	`218`	`+sem_add_executable(GenerateCLPLauncher`
`219`	`219`	`${GenerateCLPLauncher_SOURCE}`
`220`	`220`	`)`
`221`	`221`	`list(APPEND targets_to_export GenerateCLPLauncher)`