add CHUNKED encoding support too http client (#1324)

HTTP Client - fix examples
increase default timeout to 5000ms

Author: Links2004

libraries/ESP8266HTTPClient/examples/Authorization/Authorization.ino

@@ -63,7 +63,7 @@ void loop() {
         int httpCode = http.GET();
 
         // httpCode will be negative on error
-        if(httpCode) {
+        if(httpCode > 0) {
             // HTTP header has been send and Server response header has been handled
             USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode);
 

libraries/ESP8266HTTPClient/examples/BasicHttpClient/BasicHttpClient.ino

@@ -51,7 +51,7 @@ void loop() {
         int httpCode = http.GET();
 
         // httpCode will be negative on error
-        if(httpCode) {
+        if(httpCode > 0) {
             // HTTP header has been send and Server response header has been handled
             USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode);
 

libraries/ESP8266HTTPClient/examples/ReuseConnection/ReuseConnection.ino

@@ -48,7 +48,7 @@ void loop() {
         //http.begin("192.168.1.12", 80, "/test.html");
 
         int httpCode = http.GET();
-        if(httpCode) {
+        if(httpCode > 0) {
             USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode);
 
             // file found at server

libraries/ESP8266HTTPClient/examples/StreamHttpClient/StreamHttpClient.ino

@@ -50,7 +50,7 @@ void loop() {
         USE_SERIAL.print("[HTTP] GET...\n");
         // start connection and send HTTP header
         int httpCode = http.GET();
-        if(httpCode) {
+        if(httpCode > 0) {
             // HTTP header has been send and Server response header has been handled
             USE_SERIAL.printf("[HTTP] GET... code: %d\n", httpCode);
 

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.cpp

@@ -40,6 +40,8 @@ HTTPClient::HTTPClient() {
     _port = 0;
 
     _reuse = false;
+    _tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT;
+
     _https = false;
 
     _userAgent = "ESP8266HTTPClient";
@@ -50,7 +52,7 @@ HTTPClient::HTTPClient() {
     _returnCode = 0;
     _size = -1;
     _canReuse = false;
-	_tcpTimeout = HTTPCLIENT_DEFAULT_TCP_TIMEOUT;
+    _transferEncoding = HTTPC_TE_IDENTITY;
 
 }
 
@@ -468,61 +470,36 @@ int HTTPClient::writeToStream(Stream * stream) {
 
     // get length of document (is -1 when Server sends no Content-Length header)
     int len = _size;
-    int bytesWritten = 0;
-
-    size_t buff_size = HTTP_TCP_BUFFER_SIZE;
-
-    // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE
-    if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) {
-        buff_size = len;
-    }
-
-    // create buffer for read
-    uint8_t * buff = (uint8_t *) malloc(buff_size);
-
-    if(buff) {
-        // read all data from server
-        while(connected() && (len > 0 || len == -1)) {
-
-            // get available data size
-            size_t size = _tcp->available();
-
-            if(size) {
-                int c = _tcp->readBytes(buff, ((size > buff_size) ? buff_size : size));
-
-                // write it to Stream
-                int w = stream->write(buff, c);
-                bytesWritten += w;
-                if(w != c) {
-                    DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] short write asked for %d but got %d\n", c, w);
-                    break;
-                }
-
-                if(len > 0) {
-                    len -= c;
+    int ret = 0;
+
+    if(_transferEncoding == HTTPC_TE_IDENTITY) {
+        ret = writeToStreamDataBlock(stream, len);
+    } else if(_transferEncoding == HTTPC_TE_CHUNKED) {
+        while(1) {
+            String chunkHeader = _tcp->readStringUntil('\n');
+            chunkHeader.trim(); // remove \r
+
+            // read size of chunk
+            len = (uint32_t) strtol((const char *) chunkHeader.c_str(), NULL, 16);
+            DEBUG_HTTPCLIENT("[HTTP-Client] read chunk len: %d\n", len);
+            if(len > 0) {
+                int r = writeToStreamDataBlock(stream, len);
+                if(r < 0) {
+                    // error in writeToStreamDataBlock
+                    return r;
                 }
-
-                delay(0);
+                ret += r;
             } else {
-                delay(1);
+                break;
             }
+            delay(0);
         }
-
-        free(buff);
-
-        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] connection closed or file end (written: %d).\n", bytesWritten);
-
-        if(_size && _size != bytesWritten) {
-            DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] bytesWritten %d and size %d mismatch!.\n", bytesWritten, _size);
-        }
-
     } else {
-        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStream] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE);
-        return HTTPC_ERROR_TOO_LESS_RAM;
+        return HTTPC_ERROR_ENCODING;
     }
 
     end();
-    return bytesWritten;
+    return ret;
 }
 
 /**
@@ -567,6 +544,8 @@ String HTTPClient::errorToString(int error) {
             return String("no HTTP server");
         case HTTPC_ERROR_TOO_LESS_RAM:
             return String("too less ram");
+        case HTTPC_ERROR_ENCODING:
+            return String("Transfer-Encoding not supported");
         default:
             return String();
     }
@@ -706,6 +685,7 @@ bool HTTPClient::sendHeader(const char * type) {
     String header = String(type) + " " + _url + " HTTP/1.1\r\n"
             "Host: " + _host + "\r\n"
             "User-Agent: " + _userAgent + "\r\n"
+            "Accept-Encoding: identity;q=1 chunked;q=0.1 *;q=0\r\n"
             "Connection: ";
 
     if(_reuse) {
@@ -733,9 +713,10 @@ int HTTPClient::handleHeaderResponse() {
     if(!connected()) {
         return HTTPC_ERROR_NOT_CONNECTED;
     }
-
+    String transferEncoding;
     _returnCode = -1;
     _size = -1;
+    _transferEncoding = HTTPC_TE_IDENTITY;
 
     while(connected()) {
         size_t len = _tcp->available();
@@ -759,6 +740,10 @@ int HTTPClient::handleHeaderResponse() {
                     _canReuse = headerValue.equalsIgnoreCase("keep-alive");
                 }
 
+                if(headerName.equalsIgnoreCase("Transfer-Encoding")) {
+                    transferEncoding = headerValue;
+                }
+
                 for(size_t i = 0; i < _headerKeysCount; i++) {
                     if(_currentHeaders[i].key.equalsIgnoreCase(headerName)) {
                         _currentHeaders[i].value = headerValue;
@@ -769,9 +754,22 @@ int HTTPClient::handleHeaderResponse() {
 
             if(headerLine == "") {
                 DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] code: %d\n", _returnCode);
-                if(_size) {
+
+                if(_size > 0) {
                     DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] size: %d\n", _size);
                 }
+
+                if(transferEncoding.length() > 0) {
+                    DEBUG_HTTPCLIENT("[HTTP-Client][handleHeaderResponse] Transfer-Encoding: %s\n", transferEncoding.c_str());
+                    if(transferEncoding.equalsIgnoreCase("chunked")) {
+                        _transferEncoding = HTTPC_TE_CHUNKED;
+                    } else {
+                        return HTTPC_ERROR_ENCODING;
+                    }
+                } else {
+                    _transferEncoding = HTTPC_TE_IDENTITY;
+                }
+
                 if(_returnCode) {
                     return _returnCode;
                 } else {
@@ -787,3 +785,87 @@ int HTTPClient::handleHeaderResponse() {
 
     return HTTPC_ERROR_CONNECTION_LOST;
 }
+
+
+
+/**
+ *
+ * @param stream
+ * @param len
+ * @return
+ */
+int HTTPClient::writeToStreamDataBlock(Stream * stream, int size) {
+    int buff_size = HTTP_TCP_BUFFER_SIZE;
+    int len = size;
+    int bytesWritten = 0;
+
+    // if possible create smaller buffer then HTTP_TCP_BUFFER_SIZE
+    if((len > 0) && (len < HTTP_TCP_BUFFER_SIZE)) {
+        buff_size = len;
+    }
+
+    // create buffer for read
+    uint8_t * buff = (uint8_t *) malloc(buff_size);
+
+    if(buff) {
+        // read all data from server
+        while(connected() && (len > 0 || len == -1)) {
+
+            // get available data size
+            size_t sizeAvailable = _tcp->available();
+
+            if(sizeAvailable) {
+
+                int readBytes = sizeAvailable;
+
+                // read only the asked bytes
+                if(readBytes > len) {
+                    readBytes = len;
+                }
+
+                // not read more the buffer can handle
+                if(readBytes > buff_size) {
+                    readBytes = buff_size;
+                }
+
+                // read data
+                int c = _tcp->readBytes(buff, readBytes);
+
+                // write it to Stream
+                int w = stream->write(buff, c);
+                bytesWritten += w;
+                if(w != c) {
+                    DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] short write asked for %d but got %d\n", c, w);
+                    break;
+                }
+
+                if(stream->getWriteError()) {
+                    DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] stream write error %d\n", stream->getWriteError());
+                    break;
+                }
+
+                if(len > 0) {
+                    len -= c;
+                }
+
+                delay(0);
+            } else {
+                delay(1);
+            }
+        }
+
+        free(buff);
+
+        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] connection closed or file end (written: %d).\n", bytesWritten);
+
+        if((size > 0) && (size != bytesWritten)) {
+            DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] bytesWritten %d and size %d mismatch!.\n", bytesWritten, size);
+        }
+
+    } else {
+        DEBUG_HTTPCLIENT("[HTTP-Client][writeToStreamDataBlock] too less ram! need %d\n", HTTP_TCP_BUFFER_SIZE);
+        return HTTPC_ERROR_TOO_LESS_RAM;
+    }
+
+    return bytesWritten;
+}

libraries/ESP8266HTTPClient/src/ESP8266HTTPClient.h

@@ -25,13 +25,13 @@
 #ifndef ESP8266HTTPClient_H_
 #define ESP8266HTTPClient_H_
 
-//#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ )
+#define DEBUG_HTTPCLIENT(...) Serial1.printf( __VA_ARGS__ )
 
 #ifndef DEBUG_HTTPCLIENT
 #define DEBUG_HTTPCLIENT(...)
 #endif
 
-#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (1000)
+#define HTTPCLIENT_DEFAULT_TCP_TIMEOUT (5000)
 
 /// HTTP client errors
 #define HTTPC_ERROR_CONNECTION_REFUSED  (-1)
@@ -42,6 +42,8 @@
 #define HTTPC_ERROR_NO_STREAM           (-6)
 #define HTTPC_ERROR_NO_HTTP_SERVER      (-7)
 #define HTTPC_ERROR_TOO_LESS_RAM        (-8)
+#define HTTPC_ERROR_ENCODING            (-9)
+
 
 /// size for the stream handling
 #define HTTP_TCP_BUFFER_SIZE (1460)
@@ -108,6 +110,11 @@ typedef enum {
     HTTP_CODE_NETWORK_AUTHENTICATION_REQUIRED = 511
 } t_http_codes;
 
+typedef enum {
+    HTTPC_TE_IDENTITY,
+    HTTPC_TE_CHUNKED
+} transferEncoding_t;
+
 class HTTPClient {
     public:
         HTTPClient();
@@ -188,11 +195,13 @@ class HTTPClient {
         int _returnCode;
         int _size;
         bool _canReuse;
+        transferEncoding_t _transferEncoding;
+
 
         bool connect(void);
         bool sendHeader(const char * type);
         int handleHeaderResponse();
-
+        int writeToStreamDataBlock(Stream * stream, int len);
 };