ETag and caching for serving pre‐compressed files (#216)

* ETag and caching for serving pre‐compressed files

This PR introduces ETag handling and caching improvements for serving pre‐compressed .gz files, enabling conditional GET responses to optimize bandwidth and server resource usage.
Adds ETag header generation based on the CRC32 checksum from the gzip trailer.
Implements conditional 304 responses in the send() method when the client's ETag matches the server's.
Enhances both asynchronous file response handling and web server request processing for .gz files.

* Add files via upload

* ci(pre-commit): Apply automatic fixes

* Make _getEtag a private method

Make private: static void _getEtag(uint8_t trailer[4], char *serverETag)

---------

Co-authored-by: pre-commit-ci-lite[bot] <117423508+pre-commit-ci-lite[bot]@users.noreply.github.com>

Author: JosePineiro

src/AsyncWebServerRequest.cpp

@@ -0,0 +1,81 @@
+#include <ESPAsyncWebServer.h>
+
+/**
+ * @brief Sends a file from the filesystem to the client, with optional gzip compression and ETag-based caching.
+ *
+ * This method serves files over HTTP from the provided filesystem. If a compressed version of the file
+ * (with a `.gz` extension) exists and the `download` flag is not set, it serves the compressed file.
+ * It also handles ETag caching using the CRC32 value from the gzip trailer, responding with `304 Not Modified`
+ * if the client's `If-None-Match` header matches the generated ETag.
+ *
+ * @param fs Reference to the filesystem (SPIFFS, LittleFS, etc.).
+ * @param path Path to the file to be served.
+ * @param contentType Optional MIME type of the file to be sent.
+ *                    If contentType is "" it will be obtained from the file extension
+ * @param download If true, forces the file to be sent as a download (disables gzip compression).
+ * @param callback Optional template processor for dynamic content generation.
+ *                 Templates will not be processed in compressed files.
+ *
+ * @note If neither the file nor its compressed version exists, responds with `404 Not Found`.
+ */
+void AsyncWebServerRequest::send(FS &fs, const String &path, const char *contentType, bool download, AwsTemplateProcessor callback) {
+  const String gzPath = path + asyncsrv::T__gz;
+  const bool useCompressedVersion = !download && fs.exists(gzPath);
+
+  // If-None-Match header
+  if (useCompressedVersion && this->hasHeader(asyncsrv::T_INM)) {
+    // CRC32-based ETag of the trailer, bytes 4-7 from the end
+    File file = fs.open(gzPath, fs::FileOpenMode::read);
+    if (file && file.size() >= 18) {  // 18 is the minimum size of valid gzip file
+      file.seek(file.size() - 8);
+
+      uint8_t crcFromGzipTrailer[4];
+      if (file.read(crcFromGzipTrailer, sizeof(crcFromGzipTrailer)) == sizeof(crcFromGzipTrailer)) {
+        char serverETag[9];
+        _getEtag(crcFromGzipTrailer, serverETag);
+
+        // Compare with client's If-None-Match header
+        const AsyncWebHeader *inmHeader = this->getHeader(asyncsrv::T_INM);
+        if (inmHeader && inmHeader->value().equals(serverETag)) {
+          file.close();
+          this->send(304);  // Not Modified
+          return;
+        }
+      }
+      file.close();
+    }
+  }
+
+  // If we get here, create and send the normal response
+  if (fs.exists(path) || useCompressedVersion) {
+    send(beginResponse(fs, path, contentType, download, callback));
+  } else {
+    send(404);
+  }
+}
+
+/**
+ * @brief Generates an ETag string from a 4-byte trailer
+ *
+ * This function converts a 4-byte array into a hexadecimal ETag string enclosed in quotes.
+ *
+ * @param trailer[4] Input array of 4 bytes to convert to hexadecimal
+ * @param serverETag Output buffer to store the ETag
+ *                   Must be pre-allocated with minimum 9 bytes (8 hex + 1 null terminator)
+ */
+void AsyncWebServerRequest::_getEtag(uint8_t trailer[4], char *serverETag) {
+  static constexpr char hexChars[] = "0123456789ABCDEF";
+
+  uint32_t data;
+  memcpy(&data, trailer, 4);
+
+  serverETag[0] = hexChars[(data >> 4) & 0x0F];
+  serverETag[1] = hexChars[data & 0x0F];
+  serverETag[2] = hexChars[(data >> 12) & 0x0F];
+  serverETag[3] = hexChars[(data >> 8) & 0x0F];
+  serverETag[4] = hexChars[(data >> 20) & 0x0F];
+  serverETag[5] = hexChars[(data >> 16) & 0x0F];
+  serverETag[6] = hexChars[(data >> 28)];
+  serverETag[7] = hexChars[(data >> 24) & 0x0F];
+  serverETag[8] = '\0';
+}

src/ESPAsyncWebServer.h

@@ -203,6 +203,7 @@ class AsyncWebServerRequest {
   using FS = fs::FS;
   friend class AsyncWebServer;
   friend class AsyncCallbackWebHandler;
+  friend class AsyncFileResponse;
 
 private:
   AsyncClient *_client;
@@ -274,6 +275,8 @@ class AsyncWebServerRequest {
   void _send();
   void _runMiddlewareChain();
 
+  static void _getEtag(uint8_t trailer[4], char *serverETag);
+
 public:
   File _tempFile;
   void *_tempObject;
@@ -386,13 +389,7 @@ class AsyncWebServerRequest {
     send(beginResponse(code, contentType, content, len, callback));
   }
 
-  void send(FS &fs, const String &path, const char *contentType = asyncsrv::empty, bool download = false, AwsTemplateProcessor callback = nullptr) {
-    if (fs.exists(path) || (!download && fs.exists(path + asyncsrv::T__gz))) {
-      send(beginResponse(fs, path, contentType, download, callback));
-    } else {
-      send(404);
-    }
-  }
+  void send(FS &fs, const String &path, const char *contentType = asyncsrv::empty, bool download = false, AwsTemplateProcessor callback = nullptr);
   void send(FS &fs, const String &path, const String &contentType, bool download = false, AwsTemplateProcessor callback = nullptr) {
     send(fs, path, contentType.c_str(), download, callback);
   }

src/WebResponses.cpp

@@ -673,20 +673,38 @@ void AsyncFileResponse::_setContentTypeFromPath(const String &path) {
 AsyncFileResponse::AsyncFileResponse(FS &fs, const String &path, const char *contentType, bool download, AwsTemplateProcessor callback)
   : AsyncAbstractResponse(callback) {
   _code = 200;
-  _path = path;
+  const String gzPath = path + asyncsrv::T__gz;
 
-  if (!download && !fs.exists(_path) && fs.exists(_path + T__gz)) {
-    _path = _path + T__gz;
+  if (!download && !fs.exists(path) && fs.exists(gzPath)) {
+    _path = gzPath;
+    _content = fs.open(gzPath, fs::FileOpenMode::read);
+    _contentLength = _content.size();
     addHeader(T_Content_Encoding, T_gzip, false);
     _callback = nullptr;  // Unable to process zipped templates
     _sendContentLength = true;
     _chunked = false;
+
+    // CRC32-based ETag of the trailer, bytes 4-7 from the end
+    _content.seek(_contentLength - 8);
+    uint8_t crcInTrailer[4];
+    if (_content.read(crcInTrailer, sizeof(crcInTrailer)) == sizeof(crcInTrailer)) {
+      char serverETag[9];
+      AsyncWebServerRequest::_getEtag(crcInTrailer, serverETag);
+      addHeader(T_ETag, serverETag, false);
+      addHeader(T_Cache_Control, T_no_cache, false);
+    }
+
+    // Return to the beginning of the file
+    _content.seek(0);
   }
 
-  _content = fs.open(_path, fs::FileOpenMode::read);
-  _contentLength = _content.size();
+  if (!_content) {
+    _path = path;
+    _content = fs.open(path, fs::FileOpenMode::read);
+    _contentLength = _content.size();
+  }
 
-  if (strlen(contentType) == 0) {
+  if (*contentType != '\0') {
     _setContentTypeFromPath(path);
   } else {
     _contentType = contentType;