Skip to content

Commit c765584

Browse files
authored
Add zstd support (#2088)
* Add zstd support * Add zstd to CI tests * Use use zstd cmake target instead of ZSTD. Use cmake variable for found packages * Add missing comment for HTTPLIB_REQUIRE_ZSTD * Fix test.yaml rebase error * Use zstd::libzstd target * Add include and library paths to ZSTD args * Run clang-format * Add zstd to httplibConfig.cmake.in
1 parent 0bda3a7 commit c765584

File tree

6 files changed

+393
-8
lines changed

6 files changed

+393
-8
lines changed

.github/workflows/test.yaml

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -66,7 +66,8 @@ jobs:
6666
sudo apt-get update
6767
sudo apt-get install -y libc6-dev${{ matrix.config.arch_suffix }} libstdc++-13-dev${{ matrix.config.arch_suffix }} \
6868
libssl-dev${{ matrix.config.arch_suffix }} libcurl4-openssl-dev${{ matrix.config.arch_suffix }} \
69-
zlib1g-dev${{ matrix.config.arch_suffix }} libbrotli-dev${{ matrix.config.arch_suffix }}
69+
zlib1g-dev${{ matrix.config.arch_suffix }} libbrotli-dev${{ matrix.config.arch_suffix }} \
70+
libzstd-dev${{ matrix.config.arch_suffix }}
7071
- name: build and run tests
7172
run: cd test && make EXTRA_CXXFLAGS="${{ matrix.config.arch_flags }}"
7273
- name: run fuzz test target
@@ -126,7 +127,7 @@ jobs:
126127
- name: Setup msbuild on windows
127128
uses: microsoft/setup-msbuild@v2
128129
- name: Install vcpkg dependencies
129-
run: vcpkg install gtest curl zlib brotli
130+
run: vcpkg install gtest curl zlib brotli zstd
130131
- name: Install OpenSSL
131132
if: ${{ matrix.config.with_ssl }}
132133
run: choco install openssl
@@ -139,6 +140,7 @@ jobs:
139140
-DHTTPLIB_COMPILE=${{ matrix.config.compiled && 'ON' || 'OFF' }}
140141
-DHTTPLIB_REQUIRE_ZLIB=ON
141142
-DHTTPLIB_REQUIRE_BROTLI=ON
143+
-DHTTPLIB_REQUIRE_ZSTD=ON
142144
-DHTTPLIB_REQUIRE_OPENSSL=${{ matrix.config.with_ssl && 'ON' || 'OFF' }}
143145
- name: Build ${{ matrix.config.name }}
144146
run: cmake --build build --config Release -- /v:m /clp:ShowCommandLine

CMakeLists.txt

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,11 @@
44
* HTTPLIB_USE_OPENSSL_IF_AVAILABLE (default on)
55
* HTTPLIB_USE_ZLIB_IF_AVAILABLE (default on)
66
* HTTPLIB_USE_BROTLI_IF_AVAILABLE (default on)
7+
* HTTPLIB_USE_ZSTD_IF_AVAILABLE (default on)
78
* HTTPLIB_REQUIRE_OPENSSL (default off)
89
* HTTPLIB_REQUIRE_ZLIB (default off)
910
* HTTPLIB_REQUIRE_BROTLI (default off)
11+
* HTTPLIB_REQUIRE_ZSTD (default off)
1012
* HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN (default on)
1113
* HTTPLIB_COMPILE (default off)
1214
* HTTPLIB_INSTALL (default on)
@@ -45,6 +47,7 @@
4547
* HTTPLIB_IS_USING_OPENSSL - a bool for if OpenSSL support is enabled.
4648
* HTTPLIB_IS_USING_ZLIB - a bool for if ZLIB support is enabled.
4749
* HTTPLIB_IS_USING_BROTLI - a bool for if Brotli support is enabled.
50+
* HTTPLIB_IS_USING_ZSTD - a bool for if ZSTD support is enabled.
4851
* HTTPLIB_IS_USING_CERTS_FROM_MACOSX_KEYCHAIN - a bool for if support of loading system certs from the Apple Keychain is enabled.
4952
* HTTPLIB_IS_COMPILED - a bool for if the library is compiled, or otherwise header-only.
5053
* HTTPLIB_INCLUDE_DIR - the root path to httplib's header (e.g. /usr/include).
@@ -101,6 +104,8 @@ option(HTTPLIB_TEST "Enables testing and builds tests" OFF)
101104
option(HTTPLIB_REQUIRE_BROTLI "Requires Brotli to be found & linked, or fails build." OFF)
102105
option(HTTPLIB_USE_BROTLI_IF_AVAILABLE "Uses Brotli (if available) to enable Brotli decompression support." ON)
103106
option(HTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN "Enable feature to load system certs from the Apple Keychain." ON)
107+
option(HTTPLIB_REQUIRE_ZSTD "Requires ZSTD to be found & linked, or fails build." OFF)
108+
option(HTTPLIB_USE_ZSTD_IF_AVAILABLE "Uses ZSTD (if available) to enable zstd support." ON)
104109
# Defaults to static library
105110
option(BUILD_SHARED_LIBS "Build the library as a shared library instead of static. Has no effect if using header-only." OFF)
106111
if (BUILD_SHARED_LIBS AND WIN32 AND HTTPLIB_COMPILE)
@@ -153,6 +158,14 @@ elseif(HTTPLIB_USE_BROTLI_IF_AVAILABLE)
153158
set(HTTPLIB_IS_USING_BROTLI ${Brotli_FOUND})
154159
endif()
155160

161+
if(HTTPLIB_REQUIRE_ZSTD)
162+
find_package(zstd REQUIRED)
163+
set(HTTPLIB_IS_USING_ZSTD TRUE)
164+
elseif(HTTPLIB_USE_ZSTD_IF_AVAILABLE)
165+
find_package(zstd QUIET)
166+
set(HTTPLIB_IS_USING_ZSTD ${zstd_FOUND})
167+
endif()
168+
156169
# Used for default, common dirs that the end-user can change (if needed)
157170
# like CMAKE_INSTALL_INCLUDEDIR or CMAKE_INSTALL_DATADIR
158171
include(GNUInstallDirs)
@@ -227,6 +240,7 @@ target_link_libraries(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC}
227240
$<$<BOOL:${HTTPLIB_IS_USING_BROTLI}>:Brotli::encoder>
228241
$<$<BOOL:${HTTPLIB_IS_USING_BROTLI}>:Brotli::decoder>
229242
$<$<BOOL:${HTTPLIB_IS_USING_ZLIB}>:ZLIB::ZLIB>
243+
$<$<BOOL:${HTTPLIB_IS_USING_ZSTD}>:zstd::libzstd>
230244
$<$<BOOL:${HTTPLIB_IS_USING_OPENSSL}>:OpenSSL::SSL>
231245
$<$<BOOL:${HTTPLIB_IS_USING_OPENSSL}>:OpenSSL::Crypto>
232246
)
@@ -236,6 +250,7 @@ target_compile_definitions(${PROJECT_NAME} ${_INTERFACE_OR_PUBLIC}
236250
$<$<BOOL:${HTTPLIB_NO_EXCEPTIONS}>:CPPHTTPLIB_NO_EXCEPTIONS>
237251
$<$<BOOL:${HTTPLIB_IS_USING_BROTLI}>:CPPHTTPLIB_BROTLI_SUPPORT>
238252
$<$<BOOL:${HTTPLIB_IS_USING_ZLIB}>:CPPHTTPLIB_ZLIB_SUPPORT>
253+
$<$<BOOL:${HTTPLIB_IS_USING_ZSTD}>:CPPHTTPLIB_ZSTD_SUPPORT>
239254
$<$<BOOL:${HTTPLIB_IS_USING_OPENSSL}>:CPPHTTPLIB_OPENSSL_SUPPORT>
240255
$<$<AND:$<PLATFORM_ID:Darwin>,$<BOOL:${HTTPLIB_IS_USING_OPENSSL}>,$<BOOL:${HTTPLIB_IS_USING_CERTS_FROM_MACOSX_KEYCHAIN}>>:CPPHTTPLIB_USE_CERTS_FROM_MACOSX_KEYCHAIN>
241256
)

cmake/httplibConfig.cmake.in

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ if(@HTTPLIB_IS_USING_BROTLI@)
3535
find_dependency(Brotli COMPONENTS common encoder decoder)
3636
endif()
3737

38+
if(@HTTPLIB_IS_USING_ZSTD@)
39+
find_dependency(zstd)
40+
endif()
41+
3842
# Mildly useful for end-users
3943
# Not really recommended to be used though
4044
set_and_check(HTTPLIB_INCLUDE_DIR "@PACKAGE_CMAKE_INSTALL_FULL_INCLUDEDIR@")
@@ -46,6 +50,7 @@ set_and_check(HTTPLIB_HEADER_PATH "@PACKAGE_CMAKE_INSTALL_FULL_INCLUDEDIR@/httpl
4650
set(httplib_OpenSSL_FOUND @HTTPLIB_IS_USING_OPENSSL@)
4751
set(httplib_ZLIB_FOUND @HTTPLIB_IS_USING_ZLIB@)
4852
set(httplib_Brotli_FOUND @HTTPLIB_IS_USING_BROTLI@)
53+
set(httplib_zstd_FOUND @HTTPLIB_IS_USING_ZSTD@)
4954

5055
check_required_components(httplib)
5156

httplib.h

Lines changed: 117 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,10 @@ using socket_t = int;
312312
#include <brotli/encode.h>
313313
#endif
314314

315+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
316+
#include <zstd.h>
317+
#endif
318+
315319
/*
316320
* Declaration
317321
*/
@@ -2445,7 +2449,7 @@ ssize_t send_socket(socket_t sock, const void *ptr, size_t size, int flags);
24452449

24462450
ssize_t read_socket(socket_t sock, void *ptr, size_t size, int flags);
24472451

2448-
enum class EncodingType { None = 0, Gzip, Brotli };
2452+
enum class EncodingType { None = 0, Gzip, Brotli, Zstd };
24492453

24502454
EncodingType encoding_type(const Request &req, const Response &res);
24512455

@@ -2558,6 +2562,34 @@ class brotli_decompressor final : public decompressor {
25582562
};
25592563
#endif
25602564

2565+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
2566+
class zstd_compressor : public compressor {
2567+
public:
2568+
zstd_compressor();
2569+
~zstd_compressor();
2570+
2571+
bool compress(const char *data, size_t data_length, bool last,
2572+
Callback callback) override;
2573+
2574+
private:
2575+
ZSTD_CCtx *ctx_ = nullptr;
2576+
};
2577+
2578+
class zstd_decompressor : public decompressor {
2579+
public:
2580+
zstd_decompressor();
2581+
~zstd_decompressor();
2582+
2583+
bool is_valid() const override;
2584+
2585+
bool decompress(const char *data, size_t data_length,
2586+
Callback callback) override;
2587+
2588+
private:
2589+
ZSTD_DCtx *ctx_ = nullptr;
2590+
};
2591+
#endif
2592+
25612593
// NOTE: until the read size reaches `fixed_buffer_size`, use `fixed_buffer`
25622594
// to store data. The call can set memory on stack for performance.
25632595
class stream_line_reader {
@@ -3949,6 +3981,12 @@ inline EncodingType encoding_type(const Request &req, const Response &res) {
39493981
if (ret) { return EncodingType::Gzip; }
39503982
#endif
39513983

3984+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
3985+
// TODO: 'Accept-Encoding' has zstd, not zstd;q=0
3986+
ret = s.find("zstd") != std::string::npos;
3987+
if (ret) { return EncodingType::Zstd; }
3988+
#endif
3989+
39523990
return EncodingType::None;
39533991
}
39543992

@@ -4157,6 +4195,61 @@ inline bool brotli_decompressor::decompress(const char *data,
41574195
}
41584196
#endif
41594197

4198+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
4199+
inline zstd_compressor::zstd_compressor() {
4200+
ctx_ = ZSTD_createCCtx();
4201+
ZSTD_CCtx_setParameter(ctx_, ZSTD_c_compressionLevel, ZSTD_fast);
4202+
}
4203+
4204+
inline zstd_compressor::~zstd_compressor() { ZSTD_freeCCtx(ctx_); }
4205+
4206+
inline bool zstd_compressor::compress(const char *data, size_t data_length,
4207+
bool last, Callback callback) {
4208+
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
4209+
4210+
ZSTD_EndDirective mode = last ? ZSTD_e_end : ZSTD_e_continue;
4211+
ZSTD_inBuffer input = {data, data_length, 0};
4212+
4213+
bool finished;
4214+
do {
4215+
ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0};
4216+
size_t const remaining = ZSTD_compressStream2(ctx_, &output, &input, mode);
4217+
4218+
if (ZSTD_isError(remaining)) { return false; }
4219+
4220+
if (!callback(buff.data(), output.pos)) { return false; }
4221+
4222+
finished = last ? (remaining == 0) : (input.pos == input.size);
4223+
4224+
} while (!finished);
4225+
4226+
return true;
4227+
}
4228+
4229+
inline zstd_decompressor::zstd_decompressor() { ctx_ = ZSTD_createDCtx(); }
4230+
4231+
inline zstd_decompressor::~zstd_decompressor() { ZSTD_freeDCtx(ctx_); }
4232+
4233+
inline bool zstd_decompressor::is_valid() const { return ctx_ != nullptr; }
4234+
4235+
inline bool zstd_decompressor::decompress(const char *data, size_t data_length,
4236+
Callback callback) {
4237+
std::array<char, CPPHTTPLIB_COMPRESSION_BUFSIZ> buff{};
4238+
ZSTD_inBuffer input = {data, data_length, 0};
4239+
4240+
while (input.pos < input.size) {
4241+
ZSTD_outBuffer output = {buff.data(), CPPHTTPLIB_COMPRESSION_BUFSIZ, 0};
4242+
size_t const remaining = ZSTD_decompressStream(ctx_, &output, &input);
4243+
4244+
if (ZSTD_isError(remaining)) { return false; }
4245+
4246+
if (!callback(buff.data(), output.pos)) { return false; }
4247+
}
4248+
4249+
return true;
4250+
}
4251+
#endif
4252+
41604253
inline bool has_header(const Headers &headers, const std::string &key) {
41614254
return headers.find(key) != headers.end();
41624255
}
@@ -4397,6 +4490,13 @@ bool prepare_content_receiver(T &x, int &status,
43974490
#else
43984491
status = StatusCode::UnsupportedMediaType_415;
43994492
return false;
4493+
#endif
4494+
} else if (encoding == "zstd") {
4495+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
4496+
decompressor = detail::make_unique<zstd_decompressor>();
4497+
#else
4498+
status = StatusCode::UnsupportedMediaType_415;
4499+
return false;
44004500
#endif
44014501
}
44024502

@@ -6634,6 +6734,10 @@ Server::write_content_with_provider(Stream &strm, const Request &req,
66346734
} else if (type == detail::EncodingType::Brotli) {
66356735
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
66366736
compressor = detail::make_unique<detail::brotli_compressor>();
6737+
#endif
6738+
} else if (type == detail::EncodingType::Zstd) {
6739+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
6740+
compressor = detail::make_unique<detail::zstd_compressor>();
66376741
#endif
66386742
} else {
66396743
compressor = detail::make_unique<detail::nocompressor>();
@@ -7049,6 +7153,8 @@ inline void Server::apply_ranges(const Request &req, Response &res,
70497153
res.set_header("Content-Encoding", "gzip");
70507154
} else if (type == detail::EncodingType::Brotli) {
70517155
res.set_header("Content-Encoding", "br");
7156+
} else if (type == detail::EncodingType::Zstd) {
7157+
res.set_header("Content-Encoding", "zstd");
70527158
}
70537159
}
70547160
}
@@ -7088,6 +7194,11 @@ inline void Server::apply_ranges(const Request &req, Response &res,
70887194
#ifdef CPPHTTPLIB_BROTLI_SUPPORT
70897195
compressor = detail::make_unique<detail::brotli_compressor>();
70907196
content_encoding = "br";
7197+
#endif
7198+
} else if (type == detail::EncodingType::Zstd) {
7199+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
7200+
compressor = detail::make_unique<detail::zstd_compressor>();
7201+
content_encoding = "zstd";
70917202
#endif
70927203
}
70937204

@@ -7812,6 +7923,10 @@ inline bool ClientImpl::write_request(Stream &strm, Request &req,
78127923
#ifdef CPPHTTPLIB_ZLIB_SUPPORT
78137924
if (!accept_encoding.empty()) { accept_encoding += ", "; }
78147925
accept_encoding += "gzip, deflate";
7926+
#endif
7927+
#ifdef CPPHTTPLIB_ZSTD_SUPPORT
7928+
if (!accept_encoding.empty()) { accept_encoding += ", "; }
7929+
accept_encoding += "zstd";
78157930
#endif
78167931
req.set_header("Accept-Encoding", accept_encoding);
78177932
}
@@ -10377,4 +10492,4 @@ inline SSL_CTX *Client::ssl_context() const {
1037710492

1037810493
} // namespace httplib
1037910494

10380-
#endif // CPPHTTPLIB_HTTPLIB_H
10495+
#endif // CPPHTTPLIB_HTTPLIB_H

test/Makefile

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,10 @@ ZLIB_SUPPORT = -DCPPHTTPLIB_ZLIB_SUPPORT -lz
1818
BROTLI_DIR = $(PREFIX)/opt/brotli
1919
BROTLI_SUPPORT = -DCPPHTTPLIB_BROTLI_SUPPORT -I$(BROTLI_DIR)/include -L$(BROTLI_DIR)/lib -lbrotlicommon -lbrotlienc -lbrotlidec
2020

21-
TEST_ARGS = gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) -pthread -lcurl
21+
ZSTD_DIR = $(PREFIX)/opt/zstd
22+
ZSTD_SUPPORT = -DCPPHTTPLIB_ZSTD_SUPPORT -I$(ZSTD_DIR)/include -L$(ZSTD_DIR)/lib -lzstd
23+
24+
TEST_ARGS = gtest/src/gtest-all.cc gtest/src/gtest_main.cc -Igtest -Igtest/include $(OPENSSL_SUPPORT) $(ZLIB_SUPPORT) $(BROTLI_SUPPORT) $(ZSTD_SUPPORT) -pthread -lcurl
2225

2326
# By default, use standalone_fuzz_target_runner.
2427
# This runner does no fuzzing, but simply executes the inputs

0 commit comments

Comments
 (0)