Merge pull request #84 from zbeekman/feat-UTF-8-support-issue-35

Feat utf 8 support issue 35

Author: jacobwilliams

.gitignore

@@ -6,6 +6,8 @@ builds/
 *.obj
 *.manifest
 .sconsign.*
+files/*.json
+src/tests/jf_test_example[12].f90
 documentation/
 visual_studio_2010/*.suo
 visual_studio_2010/*.u2d

.travis.yml

@@ -13,14 +13,14 @@ env:
   matrix:
       # CMake build with unit tests, no documentation, no coverage analysis
       # Allow to fail for now until tests are fixed
-    - BUILD_SCRIPT="mkdir cmake-build && cd cmake-build && cmake -DSKIP_DOC_GEN:BOOL=TRUE .. && make -j 4 && make test"
+    - BUILD_SCRIPT="mkdir cmake-build && cd cmake-build && cmake -DSKIP_DOC_GEN:BOOL=TRUE -DENABLE_UNICODE:BOOL=TRUE .. && make -j 4 && make test"
       SPECIFIC_DEPENDS="cmake nodejs"
       JLINT="yes"
       DOCS="no"
       FoBiS="no"
 
       # build with build.sh, make documentation, run unit tests and perform coverage analysis
-    - BUILD_SCRIPT="./build.sh"
+    - BUILD_SCRIPT="./build.sh --coverage --enable-unicode"
       SPECIFIC_DEPENDS="binutils"
       JLINT="no"
       DOCS="yes"
@@ -59,7 +59,7 @@ script:
 
 after_success:
   - cd $TRAVIS_BUILD_DIR
-  - if [[ $CODE_COVERAGE == [yY]* ]]; then gcov -o lib/ src/json_module.f90 && coveralls -n -b . ; fi
+  - if [[ $CODE_COVERAGE == [yY]* ]]; then gcov -o lib/ src/json_module.F90 && coveralls -n -b . ; fi
   - git config --global user.name "TRAVIS-CI-for-$(git --no-pager show -s --format='%cn' $TRAVIS_COMMIT)"
   - git config --global user.email "$(git --no-pager show -s --format='%ce' $TRAVIS_COMMIT)"
   - ./deploy.sh #handles updating documentation for master branch as well as tags

CMakeLists.txt

@@ -41,8 +41,9 @@ set ( VERSION "${VERSION_MAJOR}.${VERSION_MINOR}.${VERSION_PATCH}" )
 #-------------------------------------
 # Collect source files for the library
 #-------------------------------------
-set ( JF_LIB_SRCS  src/json_module.f90 )
+set ( JF_LIB_SRCS  src/json_module.F90 )
 file ( GLOB JF_TEST_SRCS "src/tests/jf_test_*.f90" )
+set ( JF_TEST_UCS4_SUPPORT_SRC "${CMAKE_SOURCE_DIR}/src/tests/introspection/test_iso_10646_support.f90")
 
 #-----------------------------------------
 # Collect all the mod files into their own
@@ -83,6 +84,22 @@ else ()
     "${ABS_LIB_INSTALL_DIR}" )
 endif ()
 
+#---------------------------------------------
+# See if our compiler supports ISO 10646/UCS4
+#---------------------------------------------
+set ( ENABLE_UNICODE FLASE CACHE BOOL
+  "Enable unicode/UCS4 support" )
+if ( ENABLE_UNICODE )
+  try_run( UCS4_TEST_RUNS UCS4_TEST_COMPILES
+    ${CMAKE_BINARY_DIR}/bin ${JF_TEST_UCS4_SUPPORT_SRC} )
+  if (UCS4_TEST_RUNS EQUAL 0)
+    add_definitions (-DUSE_UCS4)
+  else ()
+    message ( WARNING
+      "Unicode support requested but ${CMAKE_Fortran_COMPILER_ID} Fortran compiler does not support 'ISO_10646' characters!" )
+  endif ()
+endif ()
+
 #---------------------------------------------
 # Build a shared and static library by default
 #---------------------------------------------

SConstruct

@@ -17,11 +17,11 @@ import sys
 env = Environment()
 
 if env['FORTRAN'] == 'gfortran':
-    env = Environment(F90FLAGS = '-O2 -fbacktrace -g -Wall -Wextra -Wno-maybe-uninitialized -pedantic -std=f2008 -J',)
+    env = Environment(F90FLAGS = '-O2 -fbacktrace -g -Wall -Wextra -Wno-maybe-uninitialized -Wno-unused-function -pedantic -std=f2008 -J',)
 elif env['FORTRAN'] == 'ifort':
     env = Environment(F90FLAGS = '-O2 -warn -stand f08 -diag-disable 7601 -traceback -module lib',)
 
-src = join('src','json_module.f90')
+src = join('src','json_module.F90')
 ar  = join('lib','libjsonfortran'+env['LIBSUFFIX'])
 sl  = join('lib','libjsonfortran'+env['SHLIBSUFFIX'])
 mod = join('lib','json_module.mod')  ## FORTRANMODSUFFIX
@@ -34,8 +34,7 @@ obj   = []
 exe   = []
 here = os.getcwd()
 os.chdir(join('src', 'tests'))
-for f in sorted(glob.glob("*")):
-    if not f.endswith('f90'): continue
+for f in sorted(glob.glob("*.[fF]90")):
     tests.append(join('src', 'tests', f))
     e = f[:-4]
     obj.append(join('src','tests',e+env['OBJSUFFIX']))

build.sh

@@ -25,20 +25,22 @@ PROJECTNAME='jsonfortran'       # project name for robodoc (example: jsonfortran
 DOCDIR='./documentation/'       # build directory for documentation
 SRCDIR='./src/'                 # library source directory
 TESTDIR='./src/tests/'          # unit test source directory
+INTROSPECDIR='./src/tests/introspection/' # pre compile configuration tests directory
+UCS4TESTCODE='test_iso_10646_support.f90'
 BINDIR='./bin/'                 # build directory for unit tests
 LIBDIR='./lib/'                 # build directory for library
-MODCODE='json_module.f90'       # json module file name
+MODCODE='json_module.F90'       # json module file name
 LIBOUT='libjsonfortran.a'       # name of json library
 
 
 # The following warning might be triggered by ifort unless explicitly silenced:
 # warning #7601: F2008 standard does not allow an internal procedure to be an actual argument procedure name. (R1214.4).
 # In the context of F2008 this is an erroneous warning.
 # See https://prd1idz.cps.intel.com/en-us/forums/topic/486629
-INTELCOMPILERFLAGS='-c -O2 -warn -stand f08 -diag-disable 7601 -traceback'
+INTELCOMPILERFLAGS='-c -O2 -warn -stand f08 -diag-disable 7601 -diag-disable 4013 -traceback'
 #INTELCOMPILERFLAGS='-c -O2 -warn -traceback -stand f08 -assume protect_parens -assume buffered_io -check all'
 
-GNUCOMPILERFLAGS='-c -O2 -fbacktrace -Wall -Wextra -Wno-maybe-uninitialized -pedantic -std=f2008'
+GNUCOMPILERFLAGS='-c -O2 -fbacktrace -Wall -Wextra -Wno-maybe-uninitialized -Wno-unused-function -pedantic -std=f2008'
 
 FCOMPILER='gnu' #Set default compiler to gfortran
 
@@ -55,7 +57,7 @@ print_usage () {
     echo -e "\n\nUsage:\n"
     echo -e "${script_name} [--compiler {intel|gnu|<other>}] [--cflags '<custom compiler flags here>']\n\
          [--coverage [{yes|no}]] [--profile [{yes|no}]] [--skip-tests [{yes|no}]]\n\
-         [--skip-documentation [{yes|no}]] [--help]"
+         [--skip-documentation [{yes|no}]] [--enable-unicode [{yes|no}]] [--help]"
     echo ""
     echo -e "Any flags that take an optional yes or no argument will default to 'yes' when no\n\
 argument is passed. Additionally, A custom compiler may be passed to the 'compiler'\n\
@@ -94,6 +96,22 @@ while [ "$#" -ge "1" ]; do # Get command line arguments while there are more lef
 	    # no good way to check that the user didn't do something questionable
 	    shift
 	    ;;
+	--enable-unicode)
+	    case $2 in
+		yes|Yes|YES)
+		    TRY_UNICODE="yes"
+		    shift
+		    ;;
+		no|No|NO)
+		    TRY_UNICODE="no"
+		    shift
+		    ;;
+		*)
+		    TRY_UNICODE="yes"
+		    # don't shift; $2 is next arg
+		    ;;
+	    esac
+	    ;;
 	--coverage) # enable coverage
 	    case $2 in
 		yes|Yes|YES)
@@ -185,17 +203,26 @@ if [[ $CODE_PROFILE == [yY]* ]]; then
 fi
 
 if [[ $FCOMPILER == custom ]]; then
+    echo "Trying to compile with custom compiler, $FC"
     CUSTOM="-fc $FC"
 fi
 
+if [[ $TRY_UNICODE == [yY]* ]]; then
+    echo "Trying to compile library with Unicode/UCS4 support"
+    FoBiS.py build -ch -compiler ${FCOMPILER} ${CUSTOM} -cflags "${FCOMPILERFLAGS}" -dbld "${BINDIR}" -s "${INTROSPECDIR}" -dmod ./ -dobj ./ -t ${UCS4TESTCODE} -o ${UCS4TESTCODE%.f90} -colors
+    if "${BINDIR}/${UCS4TESTCODE%.f90}"; then
+	DEFINES="-DUSE_UCS4 -Wunused-function"
+    fi
+fi
+
 #build the stand-alone library:
 echo ""
 echo "Building library..."
 
 # work around for FoBiS.py PR #45
 [ -d "$LIBDIR" ] || mkdir "$LIBDIR"
 
-FoBiS.py build -ch -compiler ${FCOMPILER} ${CUSTOM} -cflags "${FCOMPILERFLAGS}" ${COVERAGE} ${PROFILING} -dbld ${LIBDIR} -s ${SRCDIR} -dmod ./ -dobj ./ -t ${MODCODE} -o ${LIBOUT} -mklib static -colors
+FoBiS.py build -ch -compiler ${FCOMPILER} ${CUSTOM} -cflags "${FCOMPILERFLAGS} ${DEFINES}" ${COVERAGE} ${PROFILING} -dbld ${LIBDIR} -s ${SRCDIR} -dmod ./ -dobj ./ -t ${MODCODE} -o ${LIBOUT} -mklib static -colors
 
 #build the unit tests (uses the above library):
 if [[ $JF_SKIP_TESTS != [yY]* ]]; then
@@ -205,10 +232,10 @@ if [[ $JF_SKIP_TESTS != [yY]* ]]; then
     # FoBiS.py PR #45 work around
     [ -d "$BINDIR" ] || mkdir "$BINDIR"
 
-    for TEST in "${TESTDIR%/}"/jf_test_*.f90; do
+    for TEST in "${TESTDIR%/}"/jf_test_*.[fF]90; do
 	THIS_TEST=${TEST##*/}
-	echo "Build ${THIS_TEST%.f90}"
-	FoBiS.py build -ch -compiler ${FCOMPILER} ${CUSTOM} -cflags "${FCOMPILERFLAGS}" ${COVERAGE} ${PROFILING} -dbld ${BINDIR} -s ${TESTDIR} -i ${LIBDIR} -libs ${LIBDIR}/${LIBOUT} -dmod ./ -dobj ./ -t ${THIS_TEST} -o ${THIS_TEST%.f90} -colors
+	echo "Build ${THIS_TEST%.[fF]90}"
+	FoBiS.py build -ch -compiler ${FCOMPILER} ${CUSTOM} -cflags "${FCOMPILERFLAGS} ${DEFINES}" ${COVERAGE} ${PROFILING} -dbld ${BINDIR} -s ${TESTDIR} -i ${LIBDIR} -libs ${LIBDIR}/${LIBOUT} -dmod ./ -dobj ./ -t ${THIS_TEST} -o ${THIS_TEST%.[fF]90} -colors
     done
 else
     echo "Skip building the unit tests since \$JF_SKIP_TESTS has been set to 'true'."

files/expected-outputs/hello-world-ascii.json

@@ -0,0 +1 @@
+../inputs/hello-world-ascii.json

files/expected-outputs/hello-world-ucs4.json

@@ -0,0 +1 @@
+../inputs/hello-world-ucs4.json

files/inputs/hello-world-ascii.json

@@ -0,0 +1,23 @@
+{
+  "hello world": {
+    "Chinese": "\u4f60\u597d\u4e16\u754c",
+    "Dutch": "Hello wereld",
+    "English": "Hello world",
+    "French": "Bonjour monde",
+    "German": "Hallo Welt",
+    "Greek": "\u03b3\u03b5\u03b9\u03ac \u03c3\u03bf\u03c5 \u03ba\u03cc\u03c3\u03bc\u03bf\u03c2",
+    "Italian": "Ciao mondo",
+    "Japanese": "\u3053\u3093\u306b\u3061\u306f\u4e16\u754c",
+    "Korean": "\uc5ec\ubcf4\uc138\uc694 \uc138\uacc4",
+    "Portuguese": "Ol\u00e1 mundo",
+    "Russian": "\u0417\u0434\u0440\u0430\u0432\u0441\u0442\u0432\u0443\u043b\u0442\u0435 \u043c\u0438\u0440",
+    "Spanish": "Hola mundo",
+    "Amharic": "\u1235\u120b\u121d \u12a0\u1208\u121d",
+    "Hebrew": "\u05e9\u05dc\u05d5\u05dd \u05e2\u05d5\u05dc\u05dd",
+    "Hindi": "\u0939\u0948\u0932\u094b \u0935\u0930\u094d\u0932\u094d\u0921",
+    "Arabic": "\u0645\u0631\u062d\u0628\u0627 \u0628\u0627\u0644\u0639\u0627\u0644\u0645",
+    "Urdu": "\u06c1\u06cc\u0644\u0648 \u062f\u0646\u06cc\u0627",
+    "Thai": "\u0e2a\u0e27\u0e31\u0e2a\u0e14\u0e35\u0e0a\u0e32\u0e27\u0e42\u0e25\u0e01"
+  },
+  "UCS4 support?": "\u2713"
+}

files/inputs/hello-world-ucs4.json

@@ -0,0 +1,23 @@
+{
+  "hello world": {
+    "Chinese": "你好世界",
+    "Dutch": "Hello wereld",
+    "English": "Hello world",
+    "French": "Bonjour monde",
+    "German": "Hallo Welt",
+    "Greek": "γειά σου κόσμος",
+    "Italian": "Ciao mondo",
+    "Japanese": "こんにちは世界",
+    "Korean": "여보세요 세계",
+    "Portuguese": "Olá mundo",
+    "Russian": "Здравствулте мир",
+    "Spanish": "Hola mundo",
+    "Amharic": "ስላም አለም",
+    "Hebrew": "שלום עולם",
+    "Hindi": "हैलो वर्ल्ड",
+    "Arabic": "مرحبا بالعالم",
+    "Urdu": "ہیلو دنیا",
+    "Thai": "สวัสดีชาวโลก"
+  },
+  "UCS4 support?": "✓"
+}