working

Author: robotdan

src/main/java/io/fusionauth/http/server/io/HTTPInputStream.java

@@ -167,6 +167,16 @@ public int read(byte[] b, int off, int len) throws IOException {
     // - If we have read past the end of the current request, push those bytes back onto the InputStream.
     // - When a maximum content length has been specified, read at most one byte past the maximum.
     int maxReadLen = maximumContentLength == -1 ? len : Math.min(len, maximumContentLength - bytesRead + 1);
+
+    // TODO : Hack - This makes compression work with fixed length requests
+    if (fixedLength) {
+      // TODO : Note : The len arg for an inflater stream is the number of un-compressed bytes.
+      //               The returned number of bytes is the un-compressed bytes. So the problem here
+      //               is that the bytesRemaining value we have is the compressed size of the payload so we can't
+      //               us it to ensure we do not read past the current fixed length request. Sad.
+//      maxReadLen = Math.min(maxReadLen, (int) bytesRemaining);
+    }
+
     int read = delegate.read(b, off, maxReadLen);
 
     // TODO : Can I optionally never override here? If I am fixed length, I could change len -> maxLen., and then never pushback.
@@ -175,6 +185,10 @@ public int read(byte[] b, int off, int len) throws IOException {
 //    int maxLen = len;
 //    int read = delegate.read(b, off, maxLen);
 
+    // TODO : This is busted with compression.
+    //        bytesRemaining is calculated based upon the Content-Length.
+    //        But the bytes read from the InputStream will be the bytes read that are un-compressed.
+    //        So we can't use the bytes read to calculate pushback. This has to go below the Decompression in the chain.
     int reportBytesRead = read;
     if (fixedLength && read > 0) {
       int extraBytes = (int) (read - bytesRemaining);
@@ -254,8 +268,9 @@ private void initialize() throws IOException {
     // HTTPInputStream (this) > Pushback (delegate) > Throughput > Socket
     // HTTPInputStream (this) > Chunked (delegate) > Pushback > Throughput > Socket
 
-    // HTTPInputStream (this) > Pushback > Decompress > Throughput > Socket
-    // HTTPInputStream (this) > Pushback > Decompress > Chunked > Throughput > Socket
+    // The way it is currently coded
+    // HTTPInputStream (this) > Decompress > Pushback > Throughput > Socket
+    // HTTPInputStream (this) > Decompress > Chunked > Pushback > Throughput > Socket
 
     // TODO : Note I could leave this alone, but when we parse the header we can lower case these values and then remove the equalsIgnoreCase here?
     //        Seems like ideally we would normalize them to lowercase earlier.

src/test/java/io/fusionauth/http/BaseSocketTest.java

@@ -66,7 +66,8 @@ private void assertResponse(String request, String chunkedExtension, String resp
       var body = bodyString.repeat(((requestBufferSize / bodyString.length())) * 2);
 
       if (request.contains("Transfer-Encoding: chunked")) {
-        body = chunkItUp(body, chunkedExtension);
+        // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk.
+        body = chunkItUp(body, 100, chunkedExtension);
       }
 
       request = request.replace("{body}", body);

src/test/java/io/fusionauth/http/BaseTest.java

@@ -436,10 +436,8 @@ protected void assertHTTPResponseEquals(Socket socket, String expectedResponse)
     }
   }
 
-  protected String chunkItUp(String body, String chunkedExtension) {
+  protected String chunkItUp(String body, int chunkSize, String chunkedExtension) {
     List<String> result = new ArrayList<>();
-    // Chunk in 100 byte increments. Using a smaller chunk size to ensure we don't end up with a single chunk.
-    int chunkSize = 100;
     for (var i = 0; i < body.length(); i += chunkSize) {
       var endIndex = Math.min(i + chunkSize, body.length());
       var chunk = body.substring(i, endIndex);

src/test/java/io/fusionauth/http/CompressionTest.java

@@ -92,19 +92,23 @@ public void compress(String scheme, String contentEncoding, String acceptEncodin
       var requestEncodings = contentEncoding.toLowerCase().trim().split(",");
       for (String part : requestEncodings) {
         String encoding = part.trim();
-        payload = encoding.equals(ContentEncodings.Deflate)
-            ? deflate(payload)
-            : gzip(payload);
+        if (encoding.equals(ContentEncodings.Deflate)) {
+          payload = deflate(payload);
+        } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) {
+          payload = gzip(payload);
+        }
       }
 
       byte[] uncompressedBody = payload;
 
       // Sanity check on round trip compress/decompress
       for (int i = requestEncodings.length - 1; i >= 0; i--) {
         String encoding = requestEncodings[i].trim();
-        uncompressedBody = encoding.equals(ContentEncodings.Deflate)
-            ? inflate(uncompressedBody)
-            : ungzip(uncompressedBody);
+        if (encoding.equals(ContentEncodings.Deflate)) {
+          uncompressedBody = inflate(uncompressedBody);
+        } else if (encoding.equals(ContentEncodings.Gzip) || encoding.equals(ContentEncodings.XGzip)) {
+          uncompressedBody = ungzip(uncompressedBody);
+        }
       }
 
       assertEquals(uncompressedBody, bodyBytes);
@@ -140,17 +144,18 @@ public void compress(String scheme, String contentEncoding, String acceptEncodin
 
       assertNotNull(expectedResponseEncoding);
 
-      String result;
+      String result = null;
       InputStream responseInputStream = response.body();
 
       if (expectedResponseEncoding.isEmpty()) {
         result = new String(responseInputStream.readAllBytes());
       } else {
         assertEquals(response.headers().firstValue(Headers.ContentEncoding).orElse(null), expectedResponseEncoding);
-        result = new String(
-            expectedResponseEncoding.equals(ContentEncodings.Deflate)
-                ? new InflaterInputStream(responseInputStream).readAllBytes()
-                : new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8);
+        if (expectedResponseEncoding.equals(ContentEncodings.Deflate)) {
+          result = new String(new InflaterInputStream(responseInputStream).readAllBytes());
+        } else if (expectedResponseEncoding.equals(ContentEncodings.Gzip) || expectedResponseEncoding.equals(ContentEncodings.XGzip)) {
+          result = new String(new GZIPInputStream(responseInputStream).readAllBytes(), StandardCharsets.UTF_8);
+        }
       }
 
       assertEquals(result, Files.readString(file));

src/test/java/io/fusionauth/http/FormDataTest.java

@@ -197,6 +197,18 @@ public Builder(String scheme) {
       this.scheme = scheme;
     }
 
+    public Builder assertOptionalExceptionOnWrite(Class<? extends Exception> clazz) {
+      // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached.
+      // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes.
+      // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur.
+      if (thrownOnWrite != null) {
+        assertEquals(thrownOnWrite.getClass(), clazz);
+        assertEquals(thrownOnWrite.getMessage(), "Broken pipe");
+      }
+
+      return this;
+    }
+
     public Builder expectNoExceptionOnWrite() {
       assertNull(thrownOnWrite);
       return this;
@@ -272,7 +284,8 @@ public Builder expectResponse(String response) throws Exception {
               """;
 
           // Convert body to chunked
-          body = chunkItUp(body, null);
+          // - Using a small chunk to ensure we end up with more than one chunk.
+          body = chunkItUp(body, 100, null);
         } else {
           var contentLength = body.getBytes(StandardCharsets.UTF_8).length;
           if (contentLength > 0) {
@@ -307,18 +320,6 @@ public Builder expectResponse(String response) throws Exception {
       return this;
     }
 
-    public Builder assertOptionalExceptionOnWrite(Class<? extends Exception> clazz) {
-      // Note that this assertion really depends upon the system the test is run on, the size of the request, and the amount of data that can be cached.
-      // - So this is an optional assertion - if exception is not null, then we should be able to assert some attributes.
-      // - With the larger sizes this exception is mostly always thrown when running tests locally, but in GHA, it doesn't always occur.
-      if (thrownOnWrite != null) {
-        assertEquals(thrownOnWrite.getClass(), clazz);
-        assertEquals(thrownOnWrite.getMessage(), "Broken pipe");
-      }
-
-      return this;
-    }
-
     public Builder withBodyParameterCount(int bodyParameterCount) {
       this.bodyParameterCount = bodyParameterCount;
       return this;

src/test/java/io/fusionauth/http/io/HTTPInputStreamTest.java

@@ -16,63 +16,117 @@
 package io.fusionauth.http.io;
 
 import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.nio.charset.StandardCharsets;
 
+import io.fusionauth.http.BaseTest;
+import io.fusionauth.http.HTTPValues.ContentEncodings;
+import io.fusionauth.http.HTTPValues.Headers;
 import io.fusionauth.http.server.HTTPRequest;
 import io.fusionauth.http.server.HTTPServerConfiguration;
 import io.fusionauth.http.server.io.HTTPInputStream;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 import static org.testng.Assert.assertEquals;
 
 /**
  * @author Daniel DeGroff
  */
-public class HTTPInputStreamTest {
-  @Test
-  public void read_chunked_withPushback() throws Exception {
+public class HTTPInputStreamTest extends BaseTest {
+  @DataProvider(name = "contentEncoding")
+  public Object[][] contentEncoding() {
+    return new Object[][]{
+        {""},
+        {"gzip"},
+        {"deflate"},
+        {"gzip, deflate"},
+        {"deflate, gzip"}
+    };
+  }
+
+  @Test(dataProvider = "contentEncoding")
+  public void read_chunked_withPushback(String contentEncoding) throws Exception {
     // Ensure that when we read a chunked encoded body that the InputStream returns the correct number of bytes read even when
     // we read past the end of the current request and use the PushbackInputStream.
 
     String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty.";
     int contentLength = content.getBytes(StandardCharsets.UTF_8).length;
 
-    // Chunk the content
-    byte[] bytes = """
-        26\r
-        These pretzels are making me thirsty. \r
-        26\r
-        These pretzels are making me thirsty. \r
-        25\r
-        These pretzels are making me thirsty.\r
-        0\r
-        \r
-        GET / HTTP/1.1\r
-        """.getBytes();
+    // Chunk the content, add part of the next request
+    String chunked = chunkItUp(content, 38, null);
+    byte[] payload = chunked.getBytes(StandardCharsets.UTF_8);
+
+    // Optionally compress the payload
+    if (!contentEncoding.isEmpty()) {
+      var requestEncodings = contentEncoding.toLowerCase().trim().split(",");
+      for (String part : requestEncodings) {
+        String encoding = part.trim();
+        if (encoding.equals(ContentEncodings.Deflate)) {
+          payload = deflate(payload);
+        } else if (encoding.equals(ContentEncodings.Gzip)) {
+          payload = gzip(payload);
+        }
+      }
+    }
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    out.write(payload);
+
+    // Add part of the next request
+    out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8));
 
     HTTPRequest request = new HTTPRequest();
-    request.setHeader("Transfer-Encoding", "chunked");
+    request.setHeader(Headers.ContentEncoding, contentEncoding);
+    request.setHeader(Headers.TransferEncoding, "chunked");
 
-    assertReadWithPushback(bytes, content, contentLength, request);
+    byte[] bytes = out.toByteArray();
+    assertReadWithPushback(bytes, content, contentLength, 16, request);
   }
 
-  @Test
-  public void read_fixedLength_withPushback() throws Exception {
+  @Test(dataProvider = "contentEncoding")
+  public void read_fixedLength_withPushback(String contentEncoding) throws Exception {
     // Ensure that when we read a fixed length body that the InputStream returns the correct number of bytes read even when
     // we read past the end of the current request and use the PushbackInputStream.
 
-    String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty.";
-    int contentLength = content.getBytes(StandardCharsets.UTF_8).length;
-
     // Fixed length body with the start of the next request in the buffer
-    byte[] bytes = (content + "GET / HTTP/1.1\r\n").getBytes(StandardCharsets.UTF_8);
+    String content = "These pretzels are making me thirsty. These pretzels are making me thirsty. These pretzels are making me thirsty.";
+    byte[] payload = content.getBytes(StandardCharsets.UTF_8);
+    int contentLength = payload.length;
+
+    // Optionally compress the payload
+    if (!contentEncoding.isEmpty()) {
+      var requestEncodings = contentEncoding.toLowerCase().trim().split(",");
+      for (String part : requestEncodings) {
+        String encoding = part.trim();
+        if (encoding.equals(ContentEncodings.Deflate)) {
+          payload = deflate(payload);
+        } else if (encoding.equals(ContentEncodings.Gzip)) {
+          payload = gzip(payload);
+        }
+      }
+    }
+
+    // Content-Length must be the compressed length
+    int compressedLength = payload.length;
+
+    ByteArrayOutputStream out = new ByteArrayOutputStream();
+    out.write(payload);
+
+    // Add part of the next request
+    out.write("GET / HTTP/1.1\r\n".getBytes(StandardCharsets.UTF_8));
 
     HTTPRequest request = new HTTPRequest();
-    request.setHeader("Content-Length", contentLength + "");
+    request.setHeader(Headers.ContentEncoding, contentEncoding);
+    request.setHeader(Headers.ContentLength, compressedLength + "");
 
-    assertReadWithPushback(bytes, content, contentLength, request);
+    // body length is 113, when compressed it is 68 (gzip) or 78 (deflate)
+    // The number of bytes available is 129.
+    byte[] bytes = out.toByteArray();
+    assertReadWithPushback(bytes, content, contentLength, 16, request);
   }
 
-  private void assertReadWithPushback(byte[] bytes, String content, int contentLength, HTTPRequest request) throws Exception {
+  private void assertReadWithPushback(byte[] bytes, String content, int contentLength, int pushedBack, HTTPRequest request)
+      throws Exception {
     int bytesAvailable = bytes.length;
     HTTPServerConfiguration configuration = new HTTPServerConfiguration().withRequestBufferSize(bytesAvailable + 100);
 
@@ -83,6 +137,7 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen
     byte[] buffer = new byte[configuration.getRequestBufferSize()];
     int read = httpInputStream.read(buffer);
 
+    // TODO : Hmm.. with compression, this is returning the compressed bytes read instead of the un-compressed bytes read. WTF?
     assertEquals(read, contentLength);
     assertEquals(new String(buffer, 0, read), content);
 
@@ -91,12 +146,12 @@ private void assertReadWithPushback(byte[] bytes, String content, int contentLen
     assertEquals(secondRead, -1);
 
     // We have 16 bytes left over
-    assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), 16);
+    assertEquals(pushbackInputStream.getAvailableBufferedBytesRemaining(), pushedBack);
 
     // Next read should start at the next request
     byte[] leftOverBuffer = new byte[100];
     int leftOverRead = pushbackInputStream.read(leftOverBuffer);
-    assertEquals(leftOverRead, 16);
+    assertEquals(leftOverRead, pushedBack);
     assertEquals(new String(leftOverBuffer, 0, leftOverRead), "GET / HTTP/1.1\r\n");
   }
 }