Merge pull request #27 from SpringMT/update-zstd-for-v1.4.5

Update zstd to v1.4.5

Author: SpringMT

README.md

@@ -10,7 +10,7 @@ See https://github.com/facebook/zstd
 Fork from https://github.com/jarredholman/ruby-zstd.
 
 ## Zstd version
-v1.4.4 (https://github.com/facebook/zstd/tree/v1.4.4)
+v1.4.5 (https://github.com/facebook/zstd/tree/v1.4.5)
 
 ## Installation
 

ext/zstdruby/libzstd/Makefile

@@ -1,12 +1,24 @@
 # ################################################################
-# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
+# Copyright (c) 2015-2020, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
 # This source code is licensed under both the BSD-style license (found in the
 # LICENSE file in the root directory of this source tree) and the GPLv2 (found
 # in the COPYING file in the root directory of this source tree).
+# You may select, at your option, one of the above-listed licenses.
 # ################################################################
 
+Q = $(if $(filter 1,$(V) $(VERBOSE)),,@)
+
+# When cross-compiling from linux to windows, you might
+# need to specify this as "Windows." Fedora build fails
+# without it.
+#
+# Note: mingw-w64 build from linux to windows does not
+# fail on other tested distros (ubuntu, debian) even
+# without manually specifying the TARGET_SYSTEM.
+TARGET_SYSTEM ?= $(OS)
+
 # Version numbers
 LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
 LIBVER_MINOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MINOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
@@ -19,11 +31,10 @@ LIBVER := $(shell echo $(LIBVER_SCRIPT))
 VERSION?= $(LIBVER)
 CCVER := $(shell $(CC) --version)
 
-CPPFLAGS+= -I. -I./common -DXXH_NAMESPACE=ZSTD_
-ifeq ($(OS),Windows_NT)   # MinGW assumed
+CPPFLAGS+= -DXXH_NAMESPACE=ZSTD_
+ifeq ($(TARGET_SYSTEM),Windows_NT)   # MinGW assumed
 CPPFLAGS   += -D__USE_MINGW_ANSI_STDIO   # compatibility with %zu formatting
 endif
-CFLAGS  ?= -O3
 DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow \
             -Wstrict-aliasing=1 -Wswitch-enum -Wdeclaration-after-statement \
             -Wstrict-prototypes -Wundef -Wpointer-arith \
@@ -50,18 +61,46 @@ ifeq ($(findstring GCC,$(CCVER)),GCC)
 decompress/zstd_decompress_block.o :	CFLAGS+=-fno-tree-vectorize
 endif
 
-ZSTD_LEGACY_SUPPORT ?= 5
+# This is a helper variable that configures a bunch of other variables to new,
+# space-optimized defaults.
+ZSTD_LIB_MINIFY ?= 0
+ifneq ($(ZSTD_LIB_MINIFY), 0)
+	HAVE_CC_OZ ?= $(shell echo "" | $(CC) -Oz -x c -c - -o /dev/null 2> /dev/null && echo 1 || echo 0)
+	ZSTD_LEGACY_SUPPORT ?= 0
+	ZSTD_LIB_DEPRECATED ?= 0
+	HUF_FORCE_DECOMPRESS_X1 ?= 1
+	ZSTD_FORCE_DECOMPRESS_SHORT ?= 1
+	ZSTD_NO_INLINE ?= 1
+	ZSTD_STRIP_ERROR_STRINGS ?= 1
+	ifneq ($(HAVE_CC_OZ), 0)
+		# Some compilers (clang) support an even more space-optimized setting.
+		CFLAGS += -Oz
+	else
+		CFLAGS += -Os
+	endif
+	CFLAGS += -fno-stack-protector -fomit-frame-pointer -fno-ident \
+	          -DDYNAMIC_BMI2=0 -DNDEBUG
+else
+	CFLAGS += -O3
+endif
+
+# Modules
 ZSTD_LIB_COMPRESSION ?= 1
 ZSTD_LIB_DECOMPRESSION ?= 1
 ZSTD_LIB_DICTBUILDER ?= 1
 ZSTD_LIB_DEPRECATED ?= 1
+
+# Legacy support
+ZSTD_LEGACY_SUPPORT ?= 5
+ZSTD_LEGACY_MULTITHREADED_API ?= 0
+
+# Build size optimizations
 HUF_FORCE_DECOMPRESS_X1 ?= 0
 HUF_FORCE_DECOMPRESS_X2 ?= 0
 ZSTD_FORCE_DECOMPRESS_SHORT ?= 0
 ZSTD_FORCE_DECOMPRESS_LONG ?= 0
 ZSTD_NO_INLINE ?= 0
 ZSTD_STRIP_ERROR_STRINGS ?= 0
-ZSTD_LEGACY_MULTITHREADED_API ?= 0
 
 ifeq ($(ZSTD_LIB_COMPRESSION), 0)
 	ZSTD_LIB_DICTBUILDER = 0
@@ -121,7 +160,6 @@ ifneq ($(ZSTD_LEGACY_SUPPORT), 0)
 ifeq ($(shell test $(ZSTD_LEGACY_SUPPORT) -lt 8; echo $$?), 0)
 	ZSTD_FILES += $(shell ls legacy/*.c | $(GREP) 'v0[$(ZSTD_LEGACY_SUPPORT)-7]')
 endif
-	CPPFLAGS += -I./legacy
 endif
 CPPFLAGS  += -DZSTD_LEGACY_SUPPORT=$(ZSTD_LEGACY_SUPPORT)
 
@@ -142,54 +180,57 @@ else
 endif
 
 
-.PHONY: default all clean install uninstall
+.PHONY: default lib-all all clean install uninstall
 
 default: lib-release
 
+# alias
+lib-all: all
+
 all: lib
 
 libzstd.a: ARFLAGS = rcs
 libzstd.a: $(ZSTD_OBJ)
 	@echo compiling static library
-	@$(AR) $(ARFLAGS) $@ $^
-
-libzstd.a-mt: CPPFLAGS += -DZSTD_MULTITHREAD
-libzstd.a-mt: libzstd.a
+	$(Q)$(AR) $(ARFLAGS) $@ $^
 
-ifneq (,$(filter Windows%,$(OS)))
+ifneq (,$(filter Windows%,$(TARGET_SYSTEM)))
 
 LIBZSTD = dll\libzstd.dll
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.lib -shared $^ -o $@
+	$(CC) $(FLAGS) -DZSTD_DLL_EXPORT=1 -Wl,--out-implib,dll\libzstd.dll.a -shared $^ -o $@
 
 else
 
 LIBZSTD = libzstd.$(SHARED_EXT_VER)
 $(LIBZSTD): LDFLAGS += -shared -fPIC -fvisibility=hidden
 $(LIBZSTD): $(ZSTD_FILES)
 	@echo compiling dynamic library $(LIBVER)
-	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 	@echo creating versioned links
-	@ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf $@ libzstd.$(SHARED_EXT)
+	$(Q)ln -sf $@ libzstd.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf $@ libzstd.$(SHARED_EXT)
 
 endif
 
-
+.PHONY: libzstd
 libzstd : $(LIBZSTD)
 
-libzstd-mt : CPPFLAGS += -DZSTD_MULTITHREAD
-libzstd-mt : libzstd
+.PHONY: lib
+lib : libzstd.a libzstd
 
-lib: libzstd.a libzstd
+.PHONY: lib-mt
+%-mt : CPPFLAGS += -DZSTD_MULTITHREAD
+%-mt : LDFLAGS  += -pthread
+%-mt : %
+	@echo multi-threading build completed
 
-lib-mt: CPPFLAGS += -DZSTD_MULTITHREAD
-lib-mt: lib
+.PHONY: lib-release
+%-release : DEBUGFLAGS :=
+%-release : %
+	@echo release build completed
 
-lib-release lib-release-mt: DEBUGFLAGS :=
-lib-release: lib
-lib-release-mt: lib-mt
 
 # Special case : building library in single-thread mode _and_ without zstdmt_compress.c
 ZSTDMT_FILES = compress/zstdmt_compress.c
@@ -198,32 +239,54 @@ libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
 libzstd-nomt: $(ZSTD_NOMT_FILES)
 	@echo compiling single-thread dynamic library $(LIBVER)
 	@echo files : $(ZSTD_NOMT_FILES)
-	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+	$(Q)$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
 
 clean:
-	@$(RM) -r *.dSYM   # macOS-specific
-	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
-	@$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
-	@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
+	$(Q)$(RM) -r *.dSYM   # macOS-specific
+	$(Q)$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
+	$(Q)$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
+	$(Q)$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
 	@echo Cleaning library completed
 
 #-----------------------------------------------------------------------------
-# make install is validated only for Linux, macOS, BSD, Hurd and Solaris targets
+# make install is validated only for below listed environments
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS Haiku))
 
+all: libzstd.pc
+
 DESTDIR     ?=
 # directory variables : GNU conventions prefer lowercase
 # see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
 # support both lower and uppercase (BSD), use uppercase in script
 prefix      ?= /usr/local
 PREFIX      ?= $(prefix)
 exec_prefix ?= $(PREFIX)
-libdir      ?= $(exec_prefix)/lib
+EXEC_PREFIX ?= $(exec_prefix)
+libdir      ?= $(EXEC_PREFIX)/lib
 LIBDIR      ?= $(libdir)
 includedir  ?= $(PREFIX)/include
 INCLUDEDIR  ?= $(includedir)
 
+PCLIBDIR ?= $(shell echo "$(LIBDIR)" | sed -n -E -e "s@^$(EXEC_PREFIX)(/|$$)@@p")
+PCINCDIR ?= $(shell echo "$(INCLUDEDIR)" | sed -n -E -e "s@^$(PREFIX)(/|$$)@@p")
+
+ifeq (,$(PCLIBDIR))
+# Additional prefix check is required, since the empty string is technically a
+# valid PCLIBDIR
+ifeq (,$(shell echo "$(LIBDIR)" | sed -n -E -e "\\@^$(EXEC_PREFIX)(/|$$)@ p"))
+$(error configured libdir ($(LIBDIR)) is outside of prefix ($(PREFIX)), can't generate pkg-config file)
+endif
+endif
+
+ifeq (,$(PCINCDIR))
+# Additional prefix check is required, since the empty string is technically a
+# valid PCINCDIR
+ifeq (,$(shell echo "$(INCLUDEDIR)" | sed -n -E -e "\\@^$(PREFIX)(/|$$)@ p"))
+$(error configured includedir ($(INCLUDEDIR)) is outside of exec_prefix ($(EXEC_PREFIX)), can't generate pkg-config file)
+endif
+endif
+
 ifneq (,$(filter $(shell uname),FreeBSD NetBSD DragonFly))
 PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
@@ -243,47 +306,49 @@ INSTALL_DATA    ?= $(INSTALL) -m 644
 libzstd.pc:
 libzstd.pc: libzstd.pc.in
 	@echo creating pkgconfig
-	@sed -e 's|@PREFIX@|$(PREFIX)|' \
-             -e 's|@VERSION@|$(VERSION)|' \
-             $< >$@
+	$(Q)@sed -E -e 's|@PREFIX@|$(PREFIX)|' \
+          -e 's|@LIBDIR@|$(PCLIBDIR)|' \
+          -e 's|@INCLUDEDIR@|$(PCINCDIR)|' \
+          -e 's|@VERSION@|$(VERSION)|' \
+          $< >$@
 
 install: install-pc install-static install-shared install-includes
 	@echo zstd static and shared library installed
 
 install-pc: libzstd.pc
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
-	@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(PKGCONFIGDIR)/
+	$(Q)$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
 
 install-static: libzstd.a
 	@echo Installing static library
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
-	@$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
+	$(Q)$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
 
 install-shared: libzstd
 	@echo Installing shared library
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
-	@$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
-	@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(LIBDIR)/
+	$(Q)$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
+	$(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(Q)ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 
 install-includes:
 	@echo Installing includes
-	@$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
-	@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
-	@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
-	@$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)     # prototypes generate deprecation warnings
-	@$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
+	$(Q)$(INSTALL) -d -m 755 $(DESTDIR)$(INCLUDEDIR)/
+	$(Q)$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
+	$(Q)$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
+	$(Q)$(INSTALL_DATA) deprecated/zbuff.h $(DESTDIR)$(INCLUDEDIR)     # prototypes generate deprecation warnings
+	$(Q)$(INSTALL_DATA) dictBuilder/zdict.h $(DESTDIR)$(INCLUDEDIR)
 
 uninstall:
-	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
-	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
-	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	@$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
-	@$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h   # Deprecated streaming functions
-	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	$(Q)$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
+	$(Q)$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zbuff.h   # Deprecated streaming functions
+	$(Q)$(RM) $(DESTDIR)$(INCLUDEDIR)/zdict.h
 	@echo zstd libraries successfully uninstalled
 
 endif

ext/zstdruby/libzstd/README.md

@@ -85,28 +85,48 @@ The file structure is designed to make this selection manually achievable for an
 
 - While invoking `make libzstd`, it's possible to define build macros
         `ZSTD_LIB_COMPRESSION, ZSTD_LIB_DECOMPRESSION`, `ZSTD_LIB_DICTBUILDER`,
-        and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the corresponding features.
-        This will also disable compilation of all dependencies
-        (eg. `ZSTD_LIB_COMPRESSION=0` will also disable dictBuilder).
-
-- There are some additional build macros that can be used to minify the decoder.
-
-  Zstandard often has more than one implementation of a piece of functionality,
-  where each implementation optimizes for different scenarios. For example, the
-  Huffman decoder has complementary implementations that decode the stream one
-  symbol at a time or two symbols at a time. Zstd normally includes both (and
-  dispatches between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1`
-  or `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
+        and `ZSTD_LIB_DEPRECATED` as `0` to forgo compilation of the
+        corresponding features. This will also disable compilation of all
+        dependencies (eg. `ZSTD_LIB_COMPRESSION=0` will also disable
+        dictBuilder).
+
+- There are a number of options that can help minimize the binary size of
+  `libzstd`.
+
+  The first step is to select the components needed (using the above-described
+  `ZSTD_LIB_COMPRESSION` etc.).
+
+  The next step is to set `ZSTD_LIB_MINIFY` to `1` when invoking `make`. This
+  disables various optional components and changes the compilation flags to
+  prioritize space-saving.
+
+  Detailed options: Zstandard's code and build environment is set up by default
+  to optimize above all else for performance. In pursuit of this goal, Zstandard
+  makes significant trade-offs in code size. For example, Zstandard often has
+  more than one implementation of a particular component, with each
+  implementation optimized for different scenarios. For example, the Huffman
+  decoder has complementary implementations that decode the stream one symbol at
+  a time or two symbols at a time. Zstd normally includes both (and dispatches
+  between them at runtime), but by defining `HUF_FORCE_DECOMPRESS_X1` or
+  `HUF_FORCE_DECOMPRESS_X2`, you can force the use of one or the other, avoiding
   compilation of the other. Similarly, `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`
   and `ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG` force the compilation and use of
   only one or the other of two decompression implementations. The smallest
   binary is achieved by using `HUF_FORCE_DECOMPRESS_X1` and
-  `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT`.
+  `ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT` (implied by `ZSTD_LIB_MINIFY`).
 
   For squeezing the last ounce of size out, you can also define
   `ZSTD_NO_INLINE`, which disables inlining, and `ZSTD_STRIP_ERROR_STRINGS`,
   which removes the error messages that are otherwise returned by
-  `ZSTD_getErrorName`.
+  `ZSTD_getErrorName` (implied by `ZSTD_LIB_MINIFY`).
+
+  Finally, when integrating into your application, make sure you're doing link-
+  time optimation and unused symbol garbage collection (via some combination of,
+  e.g., `-flto`, `-ffat-lto-objects`, `-fuse-linker-plugin`,
+  `-ffunction-sections`, `-fdata-sections`, `-fmerge-all-constants`,
+  `-Wl,--gc-sections`, `-Wl,-z,norelro`, and an archiver that understands
+  the compiler's intermediate representation, e.g., `AR=gcc-ar`). Consult your
+  compiler's documentation.
 
 - While invoking `make libzstd`, the build macro `ZSTD_LEGACY_MULTITHREADED_API=1`
   will expose the deprecated `ZSTDMT` API exposed by `zstdmt_compress.h` in