Implemented new mechanism how to automatically expand internal buffer size. This fixes bug #241 and #124 issues.

Author: svobol13

src/main/java/com/jsoniter/IterImplForStreaming.java

@@ -276,12 +276,14 @@ public final static boolean loadMore(JsonIterator iter) throws IOException {
     private static boolean keepSkippedBytesThenRead(JsonIterator iter) throws IOException {
         int offset = iter.tail - iter.skipStartedAt;
         byte[] srcBuffer = iter.buf;
-        // Double the size of internal buffer
-        // TODO: Fix NegativeArraySizeException that happens if source stream doesnt return as much
-        // of output as was requested i.e. when n < iter.buf.length - offset. Anyhow doubling the buffer
-        // size seems to be pretty dangerous idea and should be either disabled or solved safely.
-        if (iter.skipStartedAt == 0 || iter.skipStartedAt < iter.tail / 2) {
-            iter.buf = new byte[iter.buf.length * 2];
+        // Check there is no unused buffer capacity
+        if (iter.buf.length - iter.tail == 0) {
+          // If auto expand buffer enabled, then create larger buffer
+          if (iter.autoExpandBufferStep > 0) {
+            iter.buf = new byte[iter.buf.length + iter.autoExpandBufferStep];
+          } else {
+            throw iter.reportError("loadMore", "buffer is full and autoexpansion is disabled");
+          }
         }
         System.arraycopy(srcBuffer, iter.skipStartedAt, iter.buf, 0, offset);
         int n = iter.in.read(iter.buf, offset, iter.buf.length - offset);

src/main/java/com/jsoniter/JsonIterator.java

@@ -21,6 +21,9 @@ public class JsonIterator implements Closeable {
     final static ValueType[] valueTypes = new ValueType[256];
     InputStream in;
     byte[] buf;
+    // Whenever buf is not large enough new one is created with size of
+    // buf.length + autoExpandBufferStep. Set to < 1 to disable auto expanding.
+    int autoExpandBufferStep;
     int head;
     int tail;
     int skipStartedAt = -1; // skip should keep bytes starting at this pos
@@ -60,13 +63,22 @@ private JsonIterator(InputStream in, byte[] buf, int head, int tail) {
         this.tail = tail;
     }
 
+    private JsonIterator(InputStream in, byte[] buf, int autoExpandBufferStep) {
+        this(in, buf, 0, 0);
+        this.autoExpandBufferStep = autoExpandBufferStep;
+    }
+
     public JsonIterator() {
         this(null, new byte[0], 0, 0);
     }
 
     public static JsonIterator parse(InputStream in, int bufSize) {
+        return parse(in, bufSize, bufSize);
+    }
+
+    public static JsonIterator parse(InputStream in, int bufSize, int autoExpandBufferStep) {
         enableStreamingSupport();
-        return new JsonIterator(in, new byte[bufSize], 0, 0);
+        return new JsonIterator(in, new byte[bufSize], autoExpandBufferStep);
     }
 
     public static JsonIterator parse(byte[] buf) {

src/test/java/com/jsoniter/IterImplForStreamingTest.java

@@ -1,10 +1,12 @@
 package com.jsoniter;
 
 import com.jsoniter.any.Any;
+import com.jsoniter.spi.JsonException;
 import java.io.IOException;
 import java.io.InputStream;
 import junit.framework.TestCase;
 import org.junit.experimental.categories.Category;
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
 
 public class IterImplForStreamingTest extends TestCase {
 
@@ -18,34 +20,64 @@ public void testReadMaxDouble() throws Exception {
 
 	@Category(StreamingCategory.class)
 	public void testLoadMore() throws IOException {
-		final String originalContent = "1234";
+		final String originalContent = "1234567890";
 		final byte[] src = ("{\"a\":\"" + originalContent + "\"}").getBytes();
-		InputStream slowStream = new InputStream() {
+
+		int initialBufferSize;
+		Any parsedString;
+		// Case #1: Data fits into initial buffer, autoresizing on
+		// Input must definitely fit into such large buffer
+		initialBufferSize = src.length * 2;
+		JsonIterator jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 512);
+		jsonIterator.readObject();
+		parsedString = jsonIterator.readAny();
+		assertEquals(originalContent, parsedString.toString());
+		// Check buffer was not expanded
+		assertEquals(initialBufferSize, jsonIterator.buf.length);
+
+		// Case #2: Data does fit into initial buffer, autoresizing off
+		initialBufferSize = originalContent.length() / 2;
+		jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, 0);
+		jsonIterator.readObject();
+		try {
+			jsonIterator.readAny();
+			fail("Expect to fail because buffer is too small.");
+		} catch (JsonException e) {
+			if (!e.getMessage().startsWith("loadMore")) {
+				throw e;
+			}
+		}
+		// Check buffer was not expanded
+		assertEquals(initialBufferSize, jsonIterator.buf.length);
+
+		// Case #3: Data does fit into initial buffer, autoresizing on
+		initialBufferSize = originalContent.length() / 2;
+		int autoExpandBufferStep = initialBufferSize * 3;
+		jsonIterator = JsonIterator.parse(getSluggishInputStream(src), initialBufferSize, autoExpandBufferStep);
+		jsonIterator.readObject();
+		parsedString = jsonIterator.readAny();
+		assertEquals(originalContent, parsedString.toString());
+		// Check buffer was expanded exactly once
+		assertEquals(initialBufferSize + autoExpandBufferStep, jsonIterator.buf.length);
+	}
+
+	private static InputStream getSluggishInputStream(final byte[] src) {
+		return new InputStream() {
 			int position = 0;
-			boolean pretendEmptyNextRead = false;
 
 			@Override
 			public int read() throws IOException {
+				throw new NotImplementedException();
+			}
+
+			@Override
+			public int read(byte[] b, int off, int len) throws IOException {
 				if (position < src.length) {
-					if (pretendEmptyNextRead) {
-						pretendEmptyNextRead = false;
-						return -1;
-					} else {
-						pretendEmptyNextRead = true;
-						return src[position++];
-					}
+					b[off] = src[position++];
+					return 1;
 				}
 				return -1;
 			}
 		};
-
-		// Input must definitely fit into such large buffer
-		final int initialBufferSize = src.length * 2;
-		JsonIterator jsonIterator = JsonIterator.parse(slowStream, initialBufferSize);
-		jsonIterator.readObject();
-		Any parsedString = jsonIterator.readAny();
-		assertEquals(originalContent, parsedString.toString());
-		// Check buffer was not expanded prematurely
-		assertEquals(initialBufferSize, jsonIterator.buf.length);
 	}
 }