From 6fd19972a99449222c02659965b1a4e47c096b47 Mon Sep 17 00:00:00 2001 From: Gregor Date: Mon, 2 Oct 2017 17:48:40 +0200 Subject: [PATCH 1/4] 1st draft to serve static files faster --- httpserver-buffer.lua | 48 +++++++++++++++++++++++++++++++++++++++++++ httpserver-static.lua | 30 ++++++++------------------- httpserver.lua | 38 +++++++++++++++++++++++++++++++--- 3 files changed, 91 insertions(+), 25 deletions(-) create mode 100644 httpserver-buffer.lua diff --git a/httpserver-buffer.lua b/httpserver-buffer.lua new file mode 100644 index 0000000..b9e7c40 --- /dev/null +++ b/httpserver-buffer.lua @@ -0,0 +1,48 @@ +-- httpserver-connection +-- Part of nodemcu-httpserver, provides a buffered connection object that can handle multiple +-- consecutive send() calls, and buffers small payloads to send once they get big. +-- For this to work, it must be used from a coroutine and owner is responsible for the final +-- flush() and for closing the connection. +-- Author: Gregor Hartmann + +local Buffer = {} + +-- parameter is the nodemcu-firmware connection +function Buffer:new() + local newInstance = {} + newInstance.size = 0 + newInstance.data = {} + + -- Returns true if there was any data to be sent. + function newInstance:getBuffer() + local buffer = table.concat(self.data, "") + self.data = {} + self.size = 0 + return buffer + end + + function newInstance:getpeer() + return "no peer" + end + + function newInstance:send(payload) + local flushThreshold = 1400 + if (not payload) then print("nop payload") end + local newSize = self.size + payload:len() + if newSize >= flushThreshold then + print("Buffer is full. Cutting off "..newSize-flushThreshold.." chars") + --STEP1: cut out piece from payload to complete threshold bytes in table + local pieceSize = flushThreshold - self.size + if pieceSize then + payload = payload:sub(1, pieceSize) + end + end + table.insert(self.data, payload) + self.size = self.size + #payload + end + + return newInstance + +end + +return Buffer diff --git a/httpserver-static.lua b/httpserver-static.lua index 10f2e70..1362c0b 100644 --- a/httpserver-static.lua +++ b/httpserver-static.lua @@ -1,27 +1,13 @@ -- httpserver-static.lua -- Part of nodemcu-httpserver, handles sending static files to client. --- Author: Marcos Kirsch +-- Author: Gregor Hartmann return function (connection, req, args) - dofile("httpserver-header.lc")(connection, 200, args.ext, args.isGzipped) - -- Send file in little chunks - local bytesRemaining = file.list()[args.file] - -- Chunks larger than 1024 don't work. - -- https://github.com/nodemcu/nodemcu-firmware/issues/1075 - local chunkSize = 1024 - local fileHandle = file.open(args.file) - while bytesRemaining > 0 do - local bytesToRead = 0 - if bytesRemaining > chunkSize then bytesToRead = chunkSize else bytesToRead = bytesRemaining end - local chunk = fileHandle:read(bytesToRead) - connection:send(chunk) - bytesRemaining = bytesRemaining - #chunk - --print(args.file .. ": Sent "..#chunk.. " bytes, " .. bytesRemaining .. " to go.") - chunk = nil - collectgarbage() - end - -- print("Finished sending: ", args.file) - fileHandle:close() - fileHandle = nil - collectgarbage() + + local buffer = dofile("httpserver-buffer.lc"):new() + dofile("httpserver-header.lc")(buffer, req.code or 200, args.ext, args.isGzipped) + -- Send header and return fileInfo + connection:send(buffer:getBuffer()) + + return { file = args.file, sent = 0} end diff --git a/httpserver.lua b/httpserver.lua index 18cba4b..bf0ce89 100644 --- a/httpserver.lua +++ b/httpserver.lua @@ -13,6 +13,7 @@ return function (port) -- We do it in a separate thread because we need to send in little chunks and wait for the onSent event -- before we can send more, or we risk overflowing the mcu's buffer. local connectionThread + local fileInfo local allowStatic = {GET=true, HEAD=true, POST=false, PUT=false, DELETE=false, TRACE=false, OPTIONS=false, CONNECT=false, PATCH=false} @@ -26,6 +27,10 @@ return function (port) end end + local function startServingStatic(connection, req, args) + fileInfo = dofile("httpserver-static.lc")(connection, req, args) + end + local function startServing(fileServeFunction, connection, req, args) connectionThread = coroutine.create(function(fileServeFunction, bufferedConnection, req, args) fileServeFunction(bufferedConnection, req, args) @@ -40,6 +45,7 @@ return function (port) local BufferedConnectionClass = dofile("httpserver-connection.lc") local bufferedConnection = BufferedConnectionClass:new(connection) + BufferedConnectionClass = nil local status, err = coroutine.resume(connectionThread, fileServeFunction, bufferedConnection, req, args) if not status then log(connection, "Error: "..err) @@ -50,7 +56,7 @@ return function (port) end end - local function handleRequest(connection, req) + local function handleRequest(connection, req, handleError) collectgarbage() local method = req.method local uri = req.uri @@ -89,13 +95,14 @@ return function (port) uri.args = {code = 405, errorString = "Method not supported", logFunction = log} fileServeFunction = dofile("httpserver-error.lc") end + uri.args = {file = uri.file, ext = uri.ext, isGzipped = uri.isGzipped} + startServingStatic(connection, req, uri.args) end end - startServing(fileServeFunction, connection, req, uri.args) end local function onReceive(connection, payload) - collectgarbage() +-- collectgarbage() local conf = dofile("httpserver-conf.lc") local auth local user = "Anonymous" @@ -161,6 +168,27 @@ return function (port) connectionThread = nil collectgarbage() end + elseif fileInfo then + local fileSize = file.list()[fileInfo.file] + -- Chunks larger than 1024 don't work. + -- https://github.com/nodemcu/nodemcu-firmware/issues/1075 + local chunkSize = 1024 + local fileHandle = file.open(fileInfo.file) + if fileSize > fileInfo.sent then + fileHandle:seek("set", fileInfo.sent) + local chunk = fileHandle:read(chunkSize) + fileHandle:close() + fileHandle = nil + fileInfo.sent = fileInfo.sent + #chunk + connection:send(chunk) + -- print(fileInfo.file .. ": Sent "..#chunk.. " bytes, " .. fileSize - fileInfo.sent .. " to go.") + chunk = nil + else + log(connection, "closing connetion", "Finished sending: "..fileInfo.file) + connection:close() + fileInfo = nil + end + collectgarbage() end end @@ -171,6 +199,10 @@ return function (port) connectionThread = nil collectgarbage() end + if fileInfo then + fileInfo = nil + collectgarbage() + end end connection:on("receive", onReceive) From 6c23463e26e45a2fd3bc16091eb8179c1addaa93 Mon Sep 17 00:00:00 2001 From: Gregor Date: Wed, 4 Oct 2017 23:10:33 +0200 Subject: [PATCH 2/4] Allow serving 5 images in a page the 6th image cannot be served as the esp does not open more than 5 connections at the same time --- httpserver.lua | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/httpserver.lua b/httpserver.lua index bf0ce89..eaeb2a4 100644 --- a/httpserver.lua +++ b/httpserver.lua @@ -90,15 +90,15 @@ return function (port) else if allowStatic[method] then uri.args = {file = uri.file, ext = uri.ext, isGzipped = uri.isGzipped} - fileServeFunction = dofile("httpserver-static.lc") + startServingStatic(connection, req, uri.args) + return else uri.args = {code = 405, errorString = "Method not supported", logFunction = log} fileServeFunction = dofile("httpserver-error.lc") end - uri.args = {file = uri.file, ext = uri.ext, isGzipped = uri.isGzipped} - startServingStatic(connection, req, uri.args) end end + startServing(fileServeFunction, connection, req, uri.args) end local function onReceive(connection, payload) @@ -172,7 +172,7 @@ return function (port) local fileSize = file.list()[fileInfo.file] -- Chunks larger than 1024 don't work. -- https://github.com/nodemcu/nodemcu-firmware/issues/1075 - local chunkSize = 1024 + local chunkSize = 512 local fileHandle = file.open(fileInfo.file) if fileSize > fileInfo.sent then fileHandle:seek("set", fileInfo.sent) From 39699c072f7cc62c28cc0dd3962d98d48af42fcf Mon Sep 17 00:00:00 2001 From: Gregor Date: Thu, 5 Oct 2017 21:13:56 +0200 Subject: [PATCH 3/4] win the prize --- http/cars.lua | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/http/cars.lua b/http/cars.lua index 47282cd..7dc61db 100644 --- a/http/cars.lua +++ b/http/cars.lua @@ -31,6 +31,10 @@ return function (connection, req, args) It works with three embedded images of cars, but the server crashes with four. Select the number of cars you want to see below.
Whoever manages to modify nodemcu-httpserver to load all four images without crashing wins a prize!

+

+ OK I guess I win the prize, as now you can load five cars.
+ Cheers HHHartmann +

choose: show one car show two cars From eab0ad915e2876b419b0e527b899e81b686c21e9 Mon Sep 17 00:00:00 2001 From: Gregor Date: Sat, 7 Oct 2017 13:29:16 +0200 Subject: [PATCH 4/4] Update comments --- httpserver-buffer.lua | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/httpserver-buffer.lua b/httpserver-buffer.lua index b9e7c40..18b344f 100644 --- a/httpserver-buffer.lua +++ b/httpserver-buffer.lua @@ -1,8 +1,8 @@ --- httpserver-connection --- Part of nodemcu-httpserver, provides a buffered connection object that can handle multiple --- consecutive send() calls, and buffers small payloads to send once they get big. --- For this to work, it must be used from a coroutine and owner is responsible for the final --- flush() and for closing the connection. +-- httpserver-buffer +-- Part of nodemcu-httpserver, provides a buffer that behaves like a connection object +-- that can handle multiple consecutive send() calls, and buffers small payloads up to 1400 bytes. +-- This is primarily user to collect the send requests done by the head script. +-- The owner is responsible to call getBuffer and send its result -- Author: Gregor Hartmann local Buffer = {}