Merge pull request #10 from SpringMT/update-zstd-version-to-1_3_2

Update zstd to v1.3.2

Author: SpringMT

README.md

@@ -9,7 +9,7 @@ See https://github.com/facebook/zstd
 Fork from https://github.com/jarredholman/ruby-zstd.
 
 ## Zstd version
-v1.3.1 (https://github.com/facebook/zstd/tree/v1.3.1)
+v1.3.2 (https://github.com/facebook/zstd/tree/v1.3.2)
 
 ## Installation
 

ext/zstdruby/libzstd/.gitignore

@@ -1,2 +1,3 @@
 # make install artefact
 libzstd.pc
+libzstd-nomt

ext/zstdruby/libzstd/Makefile

@@ -1,13 +1,11 @@
-# ##########################################################################
-# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# ################################################################
+# Copyright (c) 2015-present, Yann Collet, Facebook, Inc.
 # All rights reserved.
 #
-# This Makefile is validated for Linux, macOS, *BSD, Hurd, Solaris, MSYS2 targets
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree. An additional grant
-# of patent rights can be found in the PATENTS file in the same directory.
-# ##########################################################################
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
 
 # Version numbers
 LIBVER_MAJOR_SCRIPT:=`sed -n '/define ZSTD_VERSION_MAJOR/s/.*[[:blank:]]\([0-9][0-9]*\).*/\1/p' < ./zstd.h`
@@ -31,7 +29,7 @@ CFLAGS  += $(DEBUGFLAGS) $(MOREFLAGS)
 FLAGS    = $(CPPFLAGS) $(CFLAGS)
 
 
-ZSTD_FILES := $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c)
+ZSTD_FILES := $(sort $(wildcard common/*.c compress/*.c decompress/*.c dictBuilder/*.c deprecated/*.c))
 
 ZSTD_LEGACY_SUPPORT ?= 4
 
@@ -103,10 +101,19 @@ lib-release lib-release-mt: DEBUGFLAGS :=
 lib-release: lib
 lib-release-mt: lib-mt
 
+# Special case : building library in single-thread mode _and_ without zstdmt_compress.c
+ZSTDMT_FILES = compress/zstdmt_compress.c
+ZSTD_NOMT_FILES = $(filter-out $(ZSTDMT_FILES),$(ZSTD_FILES))
+libzstd-nomt: LDFLAGS += -shared -fPIC -fvisibility=hidden
+libzstd-nomt: $(ZSTD_NOMT_FILES)
+	@echo compiling single-thread dynamic library $(LIBVER)
+	@echo files : $(ZSTD_NOMT_FILES)
+	@$(CC) $(FLAGS) $^ $(LDFLAGS) $(SONAME_FLAGS) -o $@
+
 clean:
 	@$(RM) -r *.dSYM   # Mac OS-X specific
 	@$(RM) core *.o *.a *.gcda *.$(SHARED_EXT) *.$(SHARED_EXT).* libzstd.pc
-	@$(RM) dll/libzstd.dll dll/libzstd.lib
+	@$(RM) dll/libzstd.dll dll/libzstd.lib libzstd-nomt*
 	@$(RM) common/*.o compress/*.o decompress/*.o dictBuilder/*.o legacy/*.o deprecated/*.o
 	@echo Cleaning library completed
 
@@ -115,25 +122,32 @@ clean:
 #-----------------------------------------------------------------------------
 ifneq (,$(filter $(shell uname),Linux Darwin GNU/kFreeBSD GNU OpenBSD FreeBSD NetBSD DragonFly SunOS))
 
-ifneq (,$(filter $(shell uname),SunOS))
-INSTALL ?= ginstall
-else
-INSTALL ?= install
-endif
-
-PREFIX     ?= /usr/local
-DESTDIR    ?=
-LIBDIR     ?= $(PREFIX)/lib
-INCLUDEDIR ?= $(PREFIX)/include
+DESTDIR     ?=
+# directory variables : GNU conventions prefer lowercase
+# see https://www.gnu.org/prep/standards/html_node/Makefile-Conventions.html
+# support both lower and uppercase (BSD), use uppercase in script
+prefix      ?= /usr/local
+PREFIX      ?= $(prefix)
+exec_prefix ?= $(PREFIX)
+libdir      ?= $(exec_prefix)/lib
+LIBDIR      ?= $(libdir)
+includedir  ?= $(PREFIX)/include
+INCLUDEDIR  ?= $(includedir)
 
 ifneq (,$(filter $(shell uname),OpenBSD FreeBSD NetBSD DragonFly))
 PKGCONFIGDIR ?= $(PREFIX)/libdata/pkgconfig
 else
 PKGCONFIGDIR ?= $(LIBDIR)/pkgconfig
 endif
 
-INSTALL_LIB  ?= $(INSTALL) -m 755
-INSTALL_DATA ?= $(INSTALL) -m 644
+ifneq (,$(filter $(shell uname),SunOS))
+INSTALL ?= ginstall
+else
+INSTALL ?= install
+endif
+
+INSTALL_PROGRAM ?= $(INSTALL)
+INSTALL_DATA    ?= $(INSTALL) -m 644
 
 
 libzstd.pc:
@@ -150,9 +164,9 @@ install: libzstd.a libzstd libzstd.pc
 	@$(INSTALL_DATA) libzstd.pc $(DESTDIR)$(PKGCONFIGDIR)/
 	@echo Installing libraries
 	@$(INSTALL_DATA) libzstd.a $(DESTDIR)$(LIBDIR)
-	@$(INSTALL_LIB) libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)
-	@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	@ln -sf libzstd.$(SHARED_EXT_VER) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
+	@$(INSTALL_PROGRAM) $(LIBZSTD) $(DESTDIR)$(LIBDIR)
+	@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
+	@ln -sf $(LIBZSTD) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 	@echo Installing includes
 	@$(INSTALL_DATA) zstd.h $(DESTDIR)$(INCLUDEDIR)
 	@$(INSTALL_DATA) common/zstd_errors.h $(DESTDIR)$(INCLUDEDIR)
@@ -164,7 +178,7 @@ uninstall:
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.a
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT)
 	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_MAJOR)
-	@$(RM) $(DESTDIR)$(LIBDIR)/libzstd.$(SHARED_EXT_VER)
+	@$(RM) $(DESTDIR)$(LIBDIR)/$(LIBZSTD)
 	@$(RM) $(DESTDIR)$(PKGCONFIGDIR)/libzstd.pc
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd.h
 	@$(RM) $(DESTDIR)$(INCLUDEDIR)/zstd_errors.h

ext/zstdruby/libzstd/README.md

@@ -1,57 +1,78 @@
 Zstandard library files
 ================================
 
-The __lib__ directory contains several directories.
-Depending on target use case, it's enough to include only files from relevant directories.
+The __lib__ directory is split into several sub-directories,
+in order to make it easier to select or exclude specific features.
+
+
+#### Building
+
+`Makefile` script is provided, supporting the standard set of commands,
+directories, and variables (see https://www.gnu.org/prep/standards/html_node/Command-Variables.html).
+- `make` : generates both static and dynamic libraries
+- `make install` : install libraries in default system directories
 
 
 #### API
 
-Zstandard's stable API is exposed within [zstd.h](zstd.h),
-at the root of `lib` directory.
+Zstandard's stable API is exposed within [lib/zstd.h](zstd.h).
 
 
 #### Advanced API
 
-Some additional API may be useful if you're looking into advanced features :
-- common/error_public.h : transforms `size_t` function results into an `enum`,
-                          for precise error handling.
-- ZSTD_STATIC_LINKING_ONLY : if you define this macro _before_ including `zstd.h`,
-                          it will give access to advanced and experimental API.
+Optional advanced features are exposed via :
+
+- `lib/common/zstd_errors.h` : translates `size_t` function results
+                              into an `ZSTD_ErrorCode`, for accurate error handling.
+- `ZSTD_STATIC_LINKING_ONLY` : if this macro is defined _before_ including `zstd.h`,
+                          it unlocks access to advanced experimental API,
+                          exposed in second part of `zstd.h`.
                           These APIs shall ___never be used with dynamic library___ !
                           They are not "stable", their definition may change in the future.
                           Only static linking is allowed.
 
-#### ZSTDMT API
-
-To enable multithreaded compression within the library, invoke `make lib-mt` target.
-Prototypes are defined in header file `compress/zstdmt_compress.h`.
-When linking a program that uses ZSTDMT API against libzstd.a on a POSIX system,
-`-pthread` flag must be provided to the compiler and linker.
-Note : ZSTDMT prototypes can still be used with a library built without multithread support,
-but in this case, they will be single threaded only.
 
 #### Modular build
 
-Directory `common/` is required in all circumstances.
-You can select to support compression only, by just adding files from the `compress/` directory,
-In a similar way, you can build a decompressor-only library with the `decompress/` directory.
-
-Other optional functionalities provided are :
-
-- `dictBuilder/`  : source files to create dictionaries.
-                    The API can be consulted in `dictBuilder/zdict.h`.
-                    This module also depends on `common/` and `compress/` .
-
-- `legacy/` : source code to decompress previous versions of zstd, starting from `v0.1`.
-              This module also depends on `common/` and `decompress/` .
-              Library compilation must include directive `ZSTD_LEGACY_SUPPORT = 1` .
-              The main API can be consulted in `legacy/zstd_legacy.h`.
-              Advanced API from each version can be found in their relevant header file.
-              For example, advanced API for version `v0.4` is in `legacy/zstd_v04.h` .
-
-
-#### Using MinGW+MSYS to create DLL
+- Directory `lib/common` is always required, for all variants.
+- Compression source code lies in `lib/compress`
+- Decompression source code lies in `lib/decompress`
+- It's possible to include only `compress` or only `decompress`, they don't depend on each other.
+- `lib/dictBuilder` : makes it possible to generate dictionaries from a set of samples.
+    The API is exposed in `lib/dictBuilder/zdict.h`.
+    This module depends on both `lib/common` and `lib/compress` .
+- `lib/legacy` : source code to decompress older zstd formats, starting from `v0.1`.
+              This module depends on `lib/common` and `lib/decompress`.
+              To enable this feature, it's necessary to define `ZSTD_LEGACY_SUPPORT = 1` during compilation.
+              Typically, with `gcc`, add argument `-DZSTD_LEGACY_SUPPORT=1`.
+              Using higher number limits the number of version supported.
+              For example, `ZSTD_LEGACY_SUPPORT=2` means : "support legacy formats starting from v0.2+".
+              The API is exposed in `lib/legacy/zstd_legacy.h`.
+              Each version also provides a (dedicated) set of advanced API.
+              For example, advanced API for version `v0.4` is exposed in `lib/legacy/zstd_v04.h` .
+
+
+#### Multithreading support
+
+Multithreading is disabled by default when building with `make`.
+Enabling multithreading requires 2 conditions :
+- set macro `ZSTD_MULTITHREAD`
+- on POSIX systems : compile with pthread (`-pthread` compilation flag for `gcc` for example)
+
+Both conditions are automatically triggered by invoking `make lib-mt` target.
+Note that, when linking a POSIX program with a multithreaded version of `libzstd`,
+it's necessary to trigger `-pthread` flag during link stage.
+
+Multithreading capabilities are exposed via :
+- private API `lib/compress/zstdmt_compress.h`.
+  Symbols defined in this header are currently exposed in `libzstd`, hence usable.
+  Note however that this API is planned to be locked and remain strictly internal in the future.
+- advanced API `ZSTD_compress_generic()`, defined in `lib/zstd.h`, experimental section.
+  This API is still considered experimental, but is designed to be labelled "stable" at some point in the future.
+  It's the recommended entry point for multi-threading operations.
+
+
+#### Windows : using MinGW+MSYS to create DLL
 
 DLL can be created using MinGW+MSYS with the `make libzstd` command.
 This command creates `dll\libzstd.dll` and the import library `dll\libzstd.lib`.
@@ -67,19 +88,21 @@ file it should be linked with `dll\libzstd.dll`. For example:
 The compiled executable will require ZSTD DLL which is available at `dll\libzstd.dll`.
 
 
-#### Obsolete streaming API
+#### Deprecated API
 
-Streaming is now provided within `zstd.h`.
-Older streaming API is still available within `deprecated/zbuff.h`.
-It will be removed in a future version.
-Consider migrating code towards newer streaming API in `zstd.h`.
+Obsolete API on their way out are stored in directory `lib/deprecated`.
+At this stage, it contains older streaming prototypes, in `lib/deprecated/zbuff.h`.
+Presence in this directory is temporary.
+These prototypes will be removed in some future version.
+Consider migrating code towards supported streaming API exposed in `zstd.h`.
 
 
 #### Miscellaneous
 
 The other files are not source code. There are :
 
- - LICENSE : contains the BSD license text
- - Makefile : script to compile or install zstd library (static and dynamic)
- - libzstd.pc.in : for pkg-config (`make install`)
- - README.md : this file
+ - `LICENSE` : contains the BSD license text
+ - `Makefile` : `make` script to build and install zstd library (static and dynamic)
+ - `BUCK` : support for `buck` build system (https://buckbuild.com/)
+ - `libzstd.pc.in` : for `pkg-config` (used in `make install`)
+ - `README.md` : this file

ext/zstdruby/libzstd/common/bitstream.h

@@ -169,33 +169,39 @@ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits);
 ****************************************************************/
 MEM_STATIC unsigned BIT_highbit32 (register U32 val)
 {
+    assert(val != 0);
+    {
 #   if defined(_MSC_VER)   /* Visual */
-    unsigned long r=0;
-    _BitScanReverse ( &r, val );
-    return (unsigned) r;
+        unsigned long r=0;
+        _BitScanReverse ( &r, val );
+        return (unsigned) r;
 #   elif defined(__GNUC__) && (__GNUC__ >= 3)   /* Use GCC Intrinsic */
-    return 31 - __builtin_clz (val);
+        return 31 - __builtin_clz (val);
 #   else   /* Software version */
-    static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
-                                             11, 14, 16, 18, 22, 25,  3, 30,
-                                              8, 12, 20, 28, 15, 17, 24,  7,
-                                             19, 27, 23,  6, 26,  5,  4, 31 };
-    U32 v = val;
-    v |= v >> 1;
-    v |= v >> 2;
-    v |= v >> 4;
-    v |= v >> 8;
-    v |= v >> 16;
-    return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
+        static const unsigned DeBruijnClz[32] = { 0,  9,  1, 10, 13, 21,  2, 29,
+                                                 11, 14, 16, 18, 22, 25,  3, 30,
+                                                  8, 12, 20, 28, 15, 17, 24,  7,
+                                                 19, 27, 23,  6, 26,  5,  4, 31 };
+        U32 v = val;
+        v |= v >> 1;
+        v |= v >> 2;
+        v |= v >> 4;
+        v |= v >> 8;
+        v |= v >> 16;
+        return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27];
 #   endif
+    }
 }
 
 /*=====    Local Constants   =====*/
-static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, 0x3F, 0x7F,
-                                    0xFF, 0x1FF, 0x3FF, 0x7FF, 0xFFF, 0x1FFF, 0x3FFF, 0x7FFF,
-                                    0xFFFF, 0x1FFFF, 0x3FFFF, 0x7FFFF, 0xFFFFF, 0x1FFFFF, 0x3FFFFF, 0x7FFFFF,
-                                    0xFFFFFF, 0x1FFFFFF, 0x3FFFFFF };   /* up to 26 bits */
-
+static const unsigned BIT_mask[] = {
+    0,          1,         3,         7,         0xF,       0x1F,
+    0x3F,       0x7F,      0xFF,      0x1FF,     0x3FF,     0x7FF,
+    0xFFF,      0x1FFF,    0x3FFF,    0x7FFF,    0xFFFF,    0x1FFFF,
+    0x3FFFF,    0x7FFFF,   0xFFFFF,   0x1FFFFF,  0x3FFFFF,  0x7FFFFF,
+    0xFFFFFF,   0x1FFFFFF, 0x3FFFFFF, 0x7FFFFFF, 0xFFFFFFF, 0x1FFFFFFF,
+    0x3FFFFFFF, 0x7FFFFFFF}; /* up to 31 bits */
+#define BIT_MASK_SIZE (sizeof(BIT_mask) / sizeof(BIT_mask[0]))
 
 /*-**************************************************************
 *  bitStream encoding
@@ -217,11 +223,14 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC,
 }
 
 /*! BIT_addBits() :
- *  can add up to 26 bits into `bitC`.
+ *  can add up to 31 bits into `bitC`.
  *  Note : does not check for register overflow ! */
 MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC,
                             size_t value, unsigned nbBits)
 {
+    MEM_STATIC_ASSERT(BIT_MASK_SIZE == 32);
+    assert(nbBits < BIT_MASK_SIZE);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
@@ -232,6 +241,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
                                 size_t value, unsigned nbBits)
 {
     assert((value>>nbBits) == 0);
+    assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     bitC->bitContainer |= value << bitC->bitPos;
     bitC->bitPos += nbBits;
 }
@@ -242,7 +252,7 @@ MEM_STATIC void BIT_addBitsFast(BIT_CStream_t* bitC,
 MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
-    assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
     assert(bitC->ptr <= bitC->endPtr);
@@ -258,7 +268,7 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC)
 MEM_STATIC void BIT_flushBits(BIT_CStream_t* bitC)
 {
     size_t const nbBytes = bitC->bitPos >> 3;
-    assert( bitC->bitPos <= (sizeof(bitC->bitContainer)*8) );
+    assert(bitC->bitPos < sizeof(bitC->bitContainer) * 8);
     MEM_writeLEST(bitC->ptr, bitC->bitContainer);
     bitC->ptr += nbBytes;
     if (bitC->ptr > bitC->endPtr) bitC->ptr = bitC->endPtr;
@@ -350,12 +360,14 @@ MEM_STATIC size_t BIT_getMiddleBits(size_t bitContainer, U32 const start, U32 co
 #  endif
         return _bextr_u32(bitContainer, start, nbBits);
 #else
+    assert(nbBits < BIT_MASK_SIZE);
     return (bitContainer >> start) & BIT_mask[nbBits];
 #endif
 }
 
 MEM_STATIC size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits)
 {
+    assert(nbBits < BIT_MASK_SIZE);
     return bitContainer & BIT_mask[nbBits];
 }