ENH: Allow applications use UTF-8 code page

Andras Lasso · lassoan · commit 4552b9ba5ca1 · 2020-02-18T13:41:13.000-05:00
SlicerExecutionModel_USE_UTF8 is OFF (this is the default, for backward compatibility) then standard Windows API calls expects strings to be stored in the ANSI code page configured on the system.

If SlicerExecutionModel_USE_UTF8 is ON then all std::string or char buffers are expected to contain UTF-8 encoded string. This is what was the default on Mac OSX and Linux and starting from Windows Version 1903 (May 2019 Update) it has been made available as an option on Windows as well. This is where VTK and ITK (and most other software libraries) are moving towards, too.
diff --git a/CMake/SEMFunctionAddExecutable.cmake b/CMake/SEMFunctionAddExecutable.cmake
@@ -0,0 +1,22 @@
+#! Usage:
+#! \code
+#! sem_add_executable(args)
+#! \endcode
+#!
+#! This macro adds an executable that uses UTF-8 code page
+#! if SlicerExecutionModel_USE_UTF8 is enabled.
+#!
+#! Linux and Mac OSX already uses this code page by default but on Windows
+#! it has to be set explicitly, by using a manifest file.
+#!
+#! \ingroup CMakeUtilities
+
+function(sem_add_executable)
+
+if(SlicerExecutionModel_USE_UTF8 AND WIN32)
+  add_executable(${ARGN} ${SlicerExecutionModel_CMAKE_DIR}/WindowsApplicationUseUtf8.manifest)
+else()
+  add_executable(${ARGN})
+endif()
+
+endfunction()
diff --git a/CMake/SEMMacroBuildCLI.cmake b/CMake/SEMMacroBuildCLI.cmake
@@ -156,7 +156,7 @@ macro(SEMMacroBuildCLI)
     set_target_properties(${CLP}Lib PROPERTIES LINK_FLAGS ${_cli_library_link_flags})
   endif()
 
-  add_executable(${CLP} ${LOCAL_SEM_CLI_LIBRARY_WRAPPER_CXX})
+  sem_add_executable(${CLP} ${LOCAL_SEM_CLI_LIBRARY_WRAPPER_CXX})
   set_target_properties(${CLP} PROPERTIES COMPILE_FLAGS "${cli_executable_compile_flags}")
   set(cli_executable_libraries ${CLP}Lib)
   if(DEFINED SlicerExecutionModel_EXTRA_EXECUTABLE_TARGET_LIBRARIES)
diff --git a/CMake/WindowsApplicationUseUtf8.manifest b/CMake/WindowsApplicationUseUtf8.manifest
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<assembly manifestVersion="1.0" xmlns="urn:schemas-microsoft-com:asm.v3">
+  <application>
+    <windowsSettings>
+      <activeCodePage xmlns="http://schemas.microsoft.com/SMI/2019/WindowsSettings">UTF-8</activeCodePage>
+    </windowsSettings>
+  </application>
+</assembly>
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -73,6 +73,14 @@ if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
     "MinSizeRel" "RelWithDebInfo")
 endif()
 
+#-----------------------------------------------------------------------------
+# ParameterSerializer support for ModuleDescriptionParser and GenerateCLP subprojects
+#-----------------------------------------------------------------------------
+option(SlicerExecutionModel_USE_UTF8 "Make applications use UTF-8 as code page." OFF)
+set(SlicerExecutionModel_CMAKE_DIR ${CMAKE_CURRENT_SOURCE_DIR}/CMake)
+set(GenerateCLP_USE_UTF8 ${SlicerExecutionModel_USE_UTF8})
+include(SEMFunctionAddExecutable)
+
 #-----------------------------------------------------------------------------
 # Output directories associated with ModuleDescriptionParser and GenerateCLP subprojects
 #-----------------------------------------------------------------------------
@@ -229,8 +237,10 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/UseSlicerExecutionModel.cmake.in
 set(allscripts
   CMake/FindJsonCpp.cmake
   CMake/SEMCommandLineLibraryWrapper.cxx
+  CMake/SEMFunctionAddExecutable.cmake
   CMake/SEMMacroBuildCLI.cmake
   CMake/SEMToolTestName.cxx.in
+  CMake/WindowsApplicationUseUtf8.manifest
   )
 foreach(SCRIPT ${allscripts})
   get_filename_component(_fileName ${SCRIPT} NAME)
diff --git a/GenerateCLP/CMakeLists.txt b/GenerateCLP/CMakeLists.txt
@@ -105,7 +105,7 @@ set(executable_name GenerateCLP)
 set(GENERATECLP_SOURCE
   GenerateCLP.cxx
   )
-add_executable(${executable_name} ${GENERATECLP_SOURCE})
+sem_add_executable(${executable_name} ${GENERATECLP_SOURCE})
 list(APPEND targets_to_export ${executable_name})
 
 target_link_libraries(${executable_name}
@@ -215,7 +215,7 @@ add_custom_command(TARGET GenerateCLP POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E touch ${GenerateCLPLauncher_SOURCE}
   COMMENT "Force GenerateCLPLauncher to rebuild after GenerateCLP was modified"
   )
-add_executable(GenerateCLPLauncher
+sem_add_executable(GenerateCLPLauncher
   ${GenerateCLPLauncher_SOURCE}
   )
 list(APPEND targets_to_export GenerateCLPLauncher)
diff --git a/GenerateCLP/GenerateCLPConfig.cmake.in b/GenerateCLP/GenerateCLPConfig.cmake.in
@@ -7,6 +7,8 @@ set(GenerateCLP_CMAKE_DIR "@GenerateCLP_CMAKE_DIR_CONFIG@")
 set(GenerateCLP_USE_FILE "@GenerateCLP_USE_FILE_CONFIG@")
 set(GenerateCLP_USE_JSONCPP "@GenerateCLP_USE_JSONCPP@")
 set(GenerateCLP_USE_SERIALIZER "@GenerateCLP_USE_SERIALIZER@")
+set(SlicerExecutionModel_USE_UTF8 "@SlicerExecutionModel_USE_UTF8@")
+set(SlicerExecutionModel_CMAKE_DIR "@SlicerExecutionModel_CMAKE_DIR@")
 set(TCLAP_DIR "@TCLAP_DIR@")
 set(ModuleDescriptionParser_DIR "@ModuleDescriptionParser_DIR@")
 set(ITK_DIR "@ITK_DIR_CONFIG@")
@@ -29,3 +31,5 @@ find_program(GENERATECLP_EXE
 if(NOT TARGET GenerateCLP AND EXISTS "@CMAKE_CURRENT_BINARY_DIR@/GenerateCLPTargets.cmake")
   include("@CMAKE_CURRENT_BINARY_DIR@/GenerateCLPTargets.cmake")
 endif()
+
+include("${SlicerExecutionModel_CMAKE_DIR}/SEMFunctionAddExecutable.cmake")
diff --git a/GenerateCLP/GenerateCLPConfig.h.in b/GenerateCLP/GenerateCLPConfig.h.in
@@ -29,6 +29,7 @@
 #cmakedefine GENERATECLP_USE_MD5
 #cmakedefine GenerateCLP_USE_JSONCPP
 #cmakedefine GenerateCLP_USE_SERIALIZER
+#cmakedefine SlicerExecutionModel_USE_UTF8
 
 #endif
 
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTest-Configure.cmake b/GenerateCLP/Testing/CMake/GenerateCLPTest-Configure.cmake
@@ -30,6 +30,7 @@ set(command ${CMAKE_COMMAND}
   -DCMAKE_CXX_STANDARD:STRING=${CMAKE_CXX_STANDARD}
   -DGenerateCLP_DIR:PATH=${GenerateCLP_BINARY_DIR}
   -DGenerateCLP_USE_JSONCPP:BOOL=${GenerateCLP_USE_JSONCPP}
+  -DSlicerExecutionModel_USE_UTF8:BOOL=${SlicerExecutionModel_USE_UTF8}
   -DJsonCpp_CMAKE_MODULE_PATH:PATH=${JsonCpp_CMAKE_MODULE_PATH}
   -G ${generateclp_cmake_generator} ${TEST_SOURCE_DIR})
 execute_process(
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTestMacros.cmake b/GenerateCLP/Testing/CMake/GenerateCLPTestMacros.cmake
@@ -71,7 +71,7 @@ macro(GenerateCLP_TEST_PROJECT)
   #-----------------------------------------------------------------------------
   set(${PROJECT_NAME}_SOURCE ${PROJECT_NAME}.cxx)
   GENERATECLP(${PROJECT_NAME}_SOURCE ${PROJECT_NAME}.xml)
-  add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCE})
+  sem_add_executable(${PROJECT_NAME} ${${PROJECT_NAME}_SOURCE})
   if(_additional_link_libraries)
     target_link_libraries(${PROJECT_NAME} ${_additional_link_libraries})
   endif()
diff --git a/GenerateCLP/Testing/CMake/GenerateCLPTestPrerequisites.cmake.in b/GenerateCLP/Testing/CMake/GenerateCLPTestPrerequisites.cmake.in
@@ -13,6 +13,9 @@ set(JsonCpp_LIBRARY "@JsonCpp_LIBRARY@")
 
 set(JsonCpp_CMAKE_MODULE_PATH "@SlicerExecutionModel_SOURCE_DIR@/CMake")
 
+set(SlicerExecutionModel_USE_UTF8 @SlicerExecutionModel_USE_UTF8@)
+set(SlicerExecutionModel_CMAKE_DIR "@SlicerExecutionModel_CMAKE_DIR@")
+
 # --------------------------------------------------------------------------
 # Sanity checks
 
diff --git a/GenerateCLP/Testing/CMakeLists.txt b/GenerateCLP/Testing/CMakeLists.txt
@@ -49,3 +49,10 @@ set(_previous_test "NODEPENDS")
 generateclp_add_test(CLPTestMultiple Configure)
 generateclp_add_test(CLPTestMultiple Build)
 generateclp_add_test(CLPTestMultiple Test)
+
+if(SlicerExecutionModel_USE_UTF8)
+  set(_previous_test "NODEPENDS")
+  generateclp_add_test(Utf8Example Configure)
+  generateclp_add_test(Utf8Example Build)
+  generateclp_add_test(Utf8Example Test)
+endif()
diff --git a/GenerateCLP/Testing/Utf8Example/CMakeLists.txt b/GenerateCLP/Testing/Utf8Example/CMakeLists.txt
@@ -0,0 +1,16 @@
+#-----------------------------------------------------------------------------
+cmake_minimum_required(VERSION 3.13.4...3.14.2)
+cmake_policy(VERSION 3.13.4...3.14.2)
+#-----------------------------------------------------------------------------
+# Configure & Build
+#-----------------------------------------------------------------------------
+include(${CMAKE_CURRENT_SOURCE_DIR}/../CMake/GenerateCLPTestMacros.cmake)
+GenerateCLP_TEST_PROJECT(
+  NAME Utf8Example1
+)
+
+#-----------------------------------------------------------------------------
+# Test
+#-----------------------------------------------------------------------------
+add_test(NAME SlicerCLPExample1 COMMAND $<TARGET_FILE:${PROJECT_NAME}> --help)
+add_test(NAME SlicerCLPExample2 COMMAND $<TARGET_FILE:${PROJECT_NAME}>)
diff --git a/GenerateCLP/Testing/Utf8Example/Utf8Example1.cxx b/GenerateCLP/Testing/Utf8Example/Utf8Example1.cxx
@@ -0,0 +1,135 @@
+#ifdef _WIN32
+#include "Windows.h"
+#endif
+
+#include "Utf8Example1CLP.h"
+
+#include <iostream>
+#include <fstream>
+
+#ifdef _WIN32
+
+// Helper function to get Windows version
+typedef LONG NTSTATUS, * PNTSTATUS;
+#define STATUS_SUCCESS (0x00000000)
+typedef NTSTATUS(WINAPI* RtlGetVersionPtr)(PRTL_OSVERSIONINFOW);
+RTL_OSVERSIONINFOW GetRealOSVersion()
+{
+  HMODULE hMod = ::GetModuleHandleW(L"ntdll.dll");
+  if (hMod)
+  {
+    RtlGetVersionPtr fxPtr = (RtlGetVersionPtr)::GetProcAddress(hMod, "RtlGetVersion");
+    if (fxPtr != nullptr)
+    {
+      RTL_OSVERSIONINFOW rovi = { 0 };
+      rovi.dwOSVersionInfoSize = sizeof(rovi);
+      if (STATUS_SUCCESS == fxPtr(&rovi))
+      {
+        return rovi;
+      }
+    }
+  }
+  RTL_OSVERSIONINFOW rovi = { 0 };
+  return rovi;
+}
+
+#endif
+
+int main (int argc, char *argv[])
+{
+  PARSE_ARGS;
+
+  std::cout << "inputStringEnum: " << inputStringEnum << std::endl;
+  std::cout << "inputFile: " << inputFile << std::endl;
+  std::cout << "outputFile: " << outputFile << std::endl;
+
+  // Test file writing
+  std::ofstream outputFileStream;
+  outputFileStream.open(outputFile);
+  if (!outputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for writing: " << outputFile << std::endl;
+    return 1;
+  }
+  outputFileStream << inputStringEnum << std::endl;
+  outputFileStream.close();
+
+  // Test file reading
+  std::ifstream inputFileStream;
+  inputFileStream.open(outputFile);
+  if (!inputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for reading: " << inputFile << std::endl;
+  }
+  std::string writtenReadStringEnum;
+  getline(inputFileStream, writtenReadStringEnum);
+  inputFileStream.close();
+  std::cout << "written and read inputStringEnum: " << writtenReadStringEnum << std::endl;
+
+  // Test output string
+  // Write out the return parameters in "name = value" form
+  std::ofstream rts;
+  rts.open(returnParameterFile.c_str());
+  rts << "outputString = " << inputStringEnum << std::endl;
+  rts.close();
+
+#ifdef _WIN32
+  
+  // Check current windows version before proceeding
+  RTL_OSVERSIONINFOW rovi = GetRealOSVersion();
+  std::cout << "Windows version: " << rovi.dwMajorVersion << "." << rovi.dwMinorVersion << " build " << rovi.dwBuildNumber << std::endl;
+  if (rovi.dwBuildNumber < 18362)
+  {
+    std::cout << "This Windows version does not support UTF-8 as active code page in application (minimum build number is 18362). Further testing is skipped." << std::endl;
+    return 0;
+  }
+
+  // Check that active code page is UTF-8 on Windows
+  UINT activeCodePage = GetACP();
+  std::cout << "Active code page: " << activeCodePage << std::endl;
+  if (activeCodePage != CP_UTF8)
+  {
+    std::cerr << "Error: active code page is " << activeCodePage << ", expected " << CP_UTF8 << " (UTF-8)" << std::endl;
+    return 1;
+  }
+#endif
+
+  // Check that we can create a file wit utf8 filename using standard file API
+  std::string filenameUtf8 = u8"alpha(\u03b1).txt";
+  outputFileStream.open(filenameUtf8);
+  if (!outputFileStream.is_open())
+  {
+    std::cerr << "Failed to open file for writing: " << outputFile << std::endl;
+    return 1;
+  }
+  outputFileStream << inputStringEnum << std::endl;
+  outputFileStream.close();
+
+  // Test if file is created successfully
+  // We use special wide character API on Windows to ensure we properly check for the existence of the file
+  // regardless of active code page.
+#if _WIN32
+  std::wstring filenameW;
+  filenameW += L"alpha(";
+  filenameW.push_back((wchar_t)(0x03B1));
+  filenameW += L").txt";
+  FILE* tmp = _wfopen(filenameW.c_str(), L"r");
+#else
+  FILE* tmp = fopen(filenameUtf8.c_str(), "r");
+#endif
+  if (!tmp)
+  {
+    std::cerr << "Expected file does not exist: " << filenameUtf8 << std::endl;
+    return 1;
+  }
+  fclose(tmp);
+
+  // Delete the temporary file
+#if _WIN32
+  _wunlink(filenameW.c_str());
+#else
+  unlink(utf8_str.c_str());
+#endif
+
+  return 0;
+}
diff --git a/GenerateCLP/Testing/Utf8Example/Utf8Example1.xml b/GenerateCLP/Testing/Utf8Example/Utf8Example1.xml
@@ -0,0 +1,46 @@
+<?xml version="1.0" encoding="utf-8"?>
+<executable>
+  <category>utf8</category>
+  <title>Test special characters oØo</title>
+  <description>This test checks if special characters can be used in SEM</description>
+  <version>0.1.0.$Revision$(alpha)</version>
+  <documentation-url></documentation-url>
+  <license></license>
+  <contributor>András Lassó</contributor>
+
+  <parameters>
+    <label>Section låbel with an ao</label>
+    <description>Section dæscription with an ae</description>
+    <string-enumeration>
+      <name>inputStringEnum</name>
+      <longflag>--inputStringEnum</longflag>
+      <description>Enumerated strings test</description>
+      <label>Enumerated input string</label>
+      <default>Hungarian: árvíztűrő tükörfúrógép</default>
+      <element>Hungarian: árvíztűrő tükörfúrógép</element>
+      <element>Nordic: æ (ae) Æ (AE) ä (a:) Ä (A:) ø (o|) Ø (O|) ö (o:) Ö (O:) å (ao) Å (Ao)</element>
+      <element>Japanese: 日本</element>
+    </string-enumeration>
+    <file>
+      <name>inputFile</name>
+      <longflag>--inputFile</longflag>
+      <description>Generic input file</description>
+      <label>Input file</label>
+      <default><![CDATA[input-test-日本-árvíztűrő-tükörfúrógép.txt]]></default>
+    </file>
+    <file>
+      <name>outputFile</name>
+      <longflag>--outputFile</longflag>
+      <description>Generic output file</description>
+      <label>Output file</label>
+      <default>output-test-日本-árvíztűrő-tükörfúrógép.txt</default>
+    </file>
+    <string>
+      <name>outputString</name>
+      <label>Output string</label>
+      <channel>output</channel>
+      <default>árvíztűrő 日本</default>
+      <description><![CDATA[An example of a string return type]]></description>
+    </string>
+  </parameters>
+</executable>
diff --git a/ModuleDescriptionParser/Testing/CMakeLists.txt b/ModuleDescriptionParser/Testing/CMakeLists.txt
@@ -6,7 +6,7 @@ create_test_sourcelist(Tests ${KIT}CxxTests.cxx
   ModuleDescriptionTest.cxx
   )
 
-add_executable(${KIT}CxxTests
+sem_add_executable(${KIT}CxxTests
   ModuleDescriptionTestingMacros.h
   ModuleDescriptionTestingUtilities.cxx
   ModuleDescriptionTestingUtilities.h
@@ -18,7 +18,7 @@ target_link_libraries(${KIT}CxxTests ${KIT})
 # Build Parser1Test executable
 # --------------------------------------------------------------------------
 set(PARSER1TEST_SOURCE Parser1Test.cxx )
-add_executable(Parser1Test ${PARSER1TEST_SOURCE})
+sem_add_executable(Parser1Test ${PARSER1TEST_SOURCE})
 target_link_libraries(Parser1Test ModuleDescriptionParser )
 
 # --------------------------------------------------------------------------
diff --git a/ModuleDescriptionParser/Testing/TestData/parameter-file-with-pointfile-slicer-issue2979.params b/ModuleDescriptionParser/Testing/TestData/parameter-file-with-pointfile-slicer-issue2979.params
@@ -1,2 +1,2 @@
-Input Fiducial File = input.fcsv
-Output Fiducial File = output.fcsv
+Input Fiducial File = input.fcsv
+Output Fiducial File = output.fcsv
diff --git a/SlicerExecutionModelConfig.cmake.in b/SlicerExecutionModelConfig.cmake.in
@@ -53,6 +53,10 @@ if(SlicerExecutionModel_USE_SERIALIZER AND NOT SlicerExecutionModel_USE_JSONCPP)
   Make sure SlicerExecutionModel_USE_JSONCPP is ON.")
 endif()
 
+# SlicerExecutionModel_USE_UTF8 is optional and it must be turned on at
+# build time to make sure active code page is consistent in all executables.
+set(SlicerExecutionModel_USE_UTF8 @SlicerExecutionModel_USE_UTF8@)
+
 # Set SlicerExecutionModel variables
 set(SlicerExecutionModel_INCLUDE_DIRS
   "@SlicerExecutionModel_INCLUDE_DIRS_CONFIG@"
diff --git a/UseSlicerExecutionModel.cmake.in b/UseSlicerExecutionModel.cmake.in
@@ -3,4 +3,4 @@ include_directories(BEFORE ${SlicerExecutionModel_INCLUDE_DIRS})
 set(CMAKE_MODULE_PATH ${SlicerExecutionModel_CMAKE_DIR} ${CMAKE_MODULE_PATH})
 
 include(SEMMacroBuildCLI)
-
+include(SEMFunctionAddExecutable)

Original file line number	Diff line number	Diff line change
`@@ -105,7 +105,7 @@ set(executable_name GenerateCLP)`
`105`	`105`	`set(GENERATECLP_SOURCE`
`106`	`106`	`GenerateCLP.cxx`
`107`	`107`	`)`
`108`		`-add_executable(${executable_name} ${GENERATECLP_SOURCE})`
	`108`	`+sem_add_executable(${executable_name} ${GENERATECLP_SOURCE})`
`109`	`109`	`list(APPEND targets_to_export ${executable_name})`
`110`	`110`
`111`	`111`	`target_link_libraries(${executable_name}`
`@@ -215,7 +215,7 @@ add_custom_command(TARGET GenerateCLP POST_BUILD`
`215`	`215`	`COMMAND ${CMAKE_COMMAND} -E touch ${GenerateCLPLauncher_SOURCE}`
`216`	`216`	`COMMENT "Force GenerateCLPLauncher to rebuild after GenerateCLP was modified"`
`217`	`217`	`)`
`218`		`-add_executable(GenerateCLPLauncher`
	`218`	`+sem_add_executable(GenerateCLPLauncher`
`219`	`219`	`${GenerateCLPLauncher_SOURCE}`
`220`	`220`	`)`
`221`	`221`	`list(APPEND targets_to_export GenerateCLPLauncher)`