some cleanups for chunk feature.

Author: jacobwilliams

src/json_parameters.F90

@@ -113,18 +113,19 @@ module json_parameters
         !! 6 = sign + leading 0 + decimal + 'E' + exponent sign + 1 extra
     character(kind=CDK,len=*),parameter :: int_fmt  = '(ss,I0)' !! minimum width format for integers
 
-    integer(IK),parameter :: max_integer_str_len = 256 !! maximum string length of an integer.
-                                                       !! This is totally arbitrary (any way
-                                                       !! to get the compiler to tell us this?)
+    integer(IK),parameter :: max_integer_str_len = 256_IK !! maximum string length of an integer.
+                                                          !! This is totally arbitrary (any way
+                                                          !! to get the compiler to tell us this?)
 
-    integer(IK),parameter :: chunk_size = 100_IK  !! for allocatable strings: allocate chunks of this size
+    integer(IK),parameter :: chunk_size = 256_IK  !! for allocatable strings: allocate chunks of this size
     integer(IK),parameter :: unit2str = -1_IK  !! unit number to cause stuff to be
                                                !! output to strings rather than files.
                                                !! See 9.5.6.12 in the F2003/08 standard
+    character(kind=CK,len=*),parameter :: blank_chunk = repeat(space, chunk_size) !! a blank string
 
     integer(IK),parameter :: seq_chunk_size = 256_IK !! chunk size for reading sequential files
 
-    integer(IK),parameter :: stream_chunk_size = 1000_IK!! chunk size for reading stream files
+    integer(IK),parameter :: stream_chunk_size = 256_IK !! chunk size for reading stream files
 
     integer(IK),parameter :: pushed_char_size = 10_IK !! size for `pushed_char`
                                                       !! array in [[json_core(type)]]

src/json_string_utilities.F90

@@ -345,7 +345,7 @@ subroutine escape_string(str_in, str_out, escape_solidus)
             c = str_in(i:i)    !get next character in the input string
 
             !if the string is not big enough, then add another chunk:
-            if (ipos+3>len(str_out)) str_out = str_out // repeat(space, chunk_size)
+            if (ipos+3>len(str_out)) str_out = str_out // blank_chunk
 
             select case(c)
             case(backslash)

src/json_value_module.F90

@@ -258,8 +258,9 @@ module json_value_module
 
         integer :: ichunk = 0 !! index in `chunk` for [[pop_char]]
                               !! when `use_unformatted_stream=True`
-        character(kind=CK,len=stream_chunk_size) :: chunk = CK_'' !! a chunk read from a stream file
-                                                                  !! when `use_unformatted_stream=True`
+        integer :: filesize = 0 !! the file size when when `use_unformatted_stream=True`
+        character(kind=CK,len=:),allocatable :: chunk   !! a chunk read from a stream file
+                                                        !! when `use_unformatted_stream=True`
 
         contains
 
@@ -922,8 +923,11 @@ subroutine json_initialize(me,verbose,compact_reals,&
     me%char_count   = 0
     me%line_count   = 1
     me%ipos         = 1
-    me%ichunk       = 0
-    me%chunk        = ''
+    if (use_unformatted_stream) then
+        me%filesize = 0
+        me%ichunk   = 0
+        me%chunk    = repeat(' ', stream_chunk_size) ! default chunk size
+    end if
 
 #ifdef USE_UCS4
     ! reopen stdout and stderr with utf-8 encoding
@@ -8761,6 +8765,11 @@ subroutine json_parse_file(json, file, p, unit)
 
     if (istat==0) then
 
+        if (use_unformatted_stream) then
+            ! save the file save to be read:
+            inquire(unit=iunit, size=json%filesize, iostat=istat)
+        end if
+
         ! create the value and associate the pointer
         call json_value_create(p)
 
@@ -9926,7 +9935,7 @@ subroutine parse_string(json, unit, str, string)
     character(kind=CK,len=:),allocatable :: error_message !! for string unescaping
 
     !at least return a blank string if there is a problem:
-    string = repeat(space, chunk_size)
+    string = blank_chunk
 
     if (.not. json%exception_thrown) then
 
@@ -9951,7 +9960,7 @@ subroutine parse_string(json, unit, str, string)
             else
 
                 !if the string is not big enough, then add another chunk:
-                if (ip>len(string)) string = string // repeat(space, chunk_size)
+                if (ip>len(string)) string = string // blank_chunk
 
                 !append to string:
                 string(ip:ip) = c
@@ -10067,7 +10076,7 @@ subroutine parse_number(json, unit, str, value)
 
     if (.not. json%exception_thrown) then
 
-        tmp = repeat(space, chunk_size)
+        tmp = blank_chunk
         ip = 1
         first = .true.
         is_integer = .true.  !assume it may be an integer, unless otherwise determined
@@ -10091,7 +10100,7 @@ subroutine parse_number(json, unit, str, value)
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
@@ -10101,15 +10110,15 @@ subroutine parse_number(json, unit, str, value)
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
                 case(CK_'0':CK_'9')    !valid characters for numbers
 
                     !add it to the string:
                     !tmp = tmp // c   !...original
-                    if (ip>len(tmp)) tmp = tmp // repeat(space, chunk_size)
+                    if (ip>len(tmp)) tmp = tmp // blank_chunk
                     tmp(ip:ip) = c
                     ip = ip + 1
 
@@ -10177,10 +10186,6 @@ subroutine pop_char(json,unit,str,skip_ws,skip_comments,eof,popped)
     logical(LK)              :: parsing_comment !! if we are in the process
                                                 !! of parsing a comment line
 
-    logical,parameter :: chunk_it = .true. !! if true, stream files are read in chunks,
-                                           !! rather than one character at a time.
-                                           !! this speeds up the parsing dramatically.
-
     if (.not. json%exception_thrown) then
 
         eof = .false.
@@ -10213,32 +10218,36 @@ subroutine pop_char(json,unit,str,skip_ws,skip_comments,eof,popped)
                     !read the next character:
                     if (use_unformatted_stream) then
 
-                        if (chunk_it) then
-                            ! in this case, we read the file in chunks.
-                            ! if we already have the character we need,
-                            ! then get it from the chunk. Otherwise,
-                            ! read another chunk
-
-                            if (json%ichunk<1) then
-                                json%ichunk = 0
-                                read(unit=unit,pos=json%ipos,iostat=ios) json%chunk
-                            else
-                                ios = 0
+                        ! in this case, we read the file in chunks.
+                        ! if we already have the character we need,
+                        ! then get it from the chunk. Otherwise,
+                        ! read in another chunk.
+                        if (json%ichunk<1) then
+                            ! read in a chunk:
+                            json%ichunk = 0
+                            if (json%filesize<json%ipos+len(json%chunk)-1) then
+                                ! for the last chunk, we resize
+                                ! it to the correct size:
+                                json%chunk = repeat(' ', json%filesize-json%ipos+1)
                             end if
-                            json%ichunk = json%ichunk + 1
+                            read(unit=unit,pos=json%ipos,iostat=ios) json%chunk
+                        else
+                            ios = 0
+                        end if
+                        json%ichunk = json%ichunk + 1
+                        if (json%ichunk>len(json%chunk)) then
+                            ! check this just in case
+                            ios = IOSTAT_END
+                        else
+                            ! get the next character from the chunk:
                             c = json%chunk(json%ichunk:json%ichunk)
                             if (json%ichunk==len(json%chunk)) then
-                                json%ichunk = 0 ! reset
-                            else
-                                ! we have to finish getting
-                                ! characters from this chunk:
-                                if (IS_IOSTAT_END(ios)) ios = 0
+                                json%ichunk = 0 ! reset for next chunk
                             end if
-                        else
-                            read(unit=unit,pos=json%ipos,iostat=ios) c
                         end if
 
                     else
+                        ! a formatted read:
                         read(unit=unit,fmt='(A1)',advance='NO',iostat=ios) c
                     end if
                     json%ipos = json%ipos + 1
@@ -10333,7 +10342,7 @@ subroutine push_char(json,c)
 
             !in this case, c is ignored, and we just
             !decrement the stream position counter:
-            json%ipos = json%ipos - 1
+            json%ipos   = json%ipos - 1
             json%ichunk = json%ichunk - 1
 
         else