Skip to content

Commit 745a050

Browse files
darkdragndarkdragn
committed
Add lz4io and functions to r119
1 parent 8ac9cf9 commit 745a050

File tree

8 files changed

+1899
-64
lines changed

8 files changed

+1899
-64
lines changed

‎setup.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@
2020
Extension('lz4', [
2121
'src/lz4.c',
2222
'src/lz4hc.c',
23+
'src/lz4io.c',
24+
'src/xxhash.c',
2325
'src/python-lz4.c'
2426
], extra_compile_args=[
2527
"-std=c99",

‎src/lz4hc.h‎

Lines changed: 173 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,173 @@
1-
/*
2-
LZ4 HC - High Compression Mode of LZ4
3-
Header File
4-
Copyright (C) 2011-2012, Yann Collet.
5-
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6-
7-
Redistribution and use in source and binary forms, with or without
8-
modification, are permitted provided that the following conditions are
9-
met:
10-
11-
* Redistributions of source code must retain the above copyright
12-
notice, this list of conditions and the following disclaimer.
13-
* Redistributions in binary form must reproduce the above
14-
copyright notice, this list of conditions and the following disclaimer
15-
in the documentation and/or other materials provided with the
16-
distribution.
17-
18-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19-
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20-
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21-
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22-
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23-
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24-
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25-
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26-
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27-
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29-
30-
You can contact the author at :
31-
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
32-
- LZ4 source repository : http://code.google.com/p/lz4/
33-
*/
34-
#pragma once
35-
36-
37-
#if defined (__cplusplus)
38-
extern "C" {
39-
#endif
40-
41-
42-
int LZ4_compressHC (const char* source, char* dest, int isize);
43-
44-
/*
45-
LZ4_compressHC :
46-
return : the number of bytes in compressed buffer dest
47-
note : destination buffer must be already allocated.
48-
To avoid any problem, size it to handle worst cases situations (input data not compressible)
49-
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
50-
*/
51-
52-
53-
/* Note :
54-
Decompression functions are provided within regular LZ4 source code (see "lz4.h") (BSD license)
55-
*/
56-
57-
58-
#if defined (__cplusplus)
59-
}
60-
#endif
1+
/*
2+
LZ4 HC - High Compression Mode of LZ4
3+
Header File
4+
Copyright (C) 2011-2014, Yann Collet.
5+
BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php)
6+
7+
Redistribution and use in source and binary forms, with or without
8+
modification, are permitted provided that the following conditions are
9+
met:
10+
11+
* Redistributions of source code must retain the above copyright
12+
notice, this list of conditions and the following disclaimer.
13+
* Redistributions in binary form must reproduce the above
14+
copyright notice, this list of conditions and the following disclaimer
15+
in the documentation and/or other materials provided with the
16+
distribution.
17+
18+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19+
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20+
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21+
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22+
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23+
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24+
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25+
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26+
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29+
30+
You can contact the author at :
31+
- LZ4 homepage : http://fastcompression.blogspot.com/p/lz4.html
32+
- LZ4 source repository : http://code.google.com/p/lz4/
33+
*/
34+
#pragma once
35+
36+
37+
#if defined (__cplusplus)
38+
extern "C" {
39+
#endif
40+
41+
42+
int LZ4_compressHC (const char* source, char* dest, int inputSize);
43+
/*
44+
LZ4_compressHC :
45+
return : the number of bytes in compressed buffer dest
46+
or 0 if compression fails.
47+
note : destination buffer must be already allocated.
48+
To avoid any problem, size it to handle worst cases situations (input data not compressible)
49+
Worst case size evaluation is provided by function LZ4_compressBound() (see "lz4.h")
50+
*/
51+
52+
int LZ4_compressHC_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize);
53+
/*
54+
LZ4_compress_limitedOutput() :
55+
Compress 'inputSize' bytes from 'source' into an output buffer 'dest' of maximum size 'maxOutputSize'.
56+
If it cannot achieve it, compression will stop, and result of the function will be zero.
57+
This function never writes outside of provided output buffer.
58+
59+
inputSize : Max supported value is 1 GB
60+
maxOutputSize : is maximum allowed size into the destination buffer (which must be already allocated)
61+
return : the number of output bytes written in buffer 'dest'
62+
or 0 if compression fails.
63+
*/
64+
65+
66+
int LZ4_compressHC2 (const char* source, char* dest, int inputSize, int compressionLevel);
67+
int LZ4_compressHC2_limitedOutput (const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
68+
/*
69+
Same functions as above, but with programmable 'compressionLevel'.
70+
Recommended values are between 4 and 9, although any value between 0 and 16 will work.
71+
'compressionLevel'==0 means use default 'compressionLevel' value.
72+
Values above 16 behave the same as 16.
73+
Equivalent variants exist for all other compression functions below.
74+
*/
75+
76+
/* Note :
77+
Decompression functions are provided within LZ4 source code (see "lz4.h") (BSD license)
78+
*/
79+
80+
81+
/**************************************
82+
Using an external allocation
83+
**************************************/
84+
int LZ4_sizeofStateHC(void);
85+
int LZ4_compressHC_withStateHC (void* state, const char* source, char* dest, int inputSize);
86+
int LZ4_compressHC_limitedOutput_withStateHC (void* state, const char* source, char* dest, int inputSize, int maxOutputSize);
87+
88+
int LZ4_compressHC2_withStateHC (void* state, const char* source, char* dest, int inputSize, int compressionLevel);
89+
int LZ4_compressHC2_limitedOutput_withStateHC(void* state, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
90+
91+
/*
92+
These functions are provided should you prefer to allocate memory for compression tables with your own allocation methods.
93+
To know how much memory must be allocated for the compression tables, use :
94+
int LZ4_sizeofStateHC();
95+
96+
Note that tables must be aligned for pointer (32 or 64 bits), otherwise compression will fail (return code 0).
97+
98+
The allocated memory can be provided to the compressions functions using 'void* state' parameter.
99+
LZ4_compress_withStateHC() and LZ4_compress_limitedOutput_withStateHC() are equivalent to previously described functions.
100+
They just use the externally allocated memory area instead of allocating their own (on stack, or on heap).
101+
*/
102+
103+
104+
/**************************************
105+
Streaming Functions
106+
**************************************/
107+
/* Note : these streaming functions still follows the older model */
108+
void* LZ4_createHC (const char* inputBuffer);
109+
int LZ4_compressHC_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize);
110+
int LZ4_compressHC_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize);
111+
char* LZ4_slideInputBufferHC (void* LZ4HC_Data);
112+
int LZ4_freeHC (void* LZ4HC_Data);
113+
114+
int LZ4_compressHC2_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int compressionLevel);
115+
int LZ4_compressHC2_limitedOutput_continue (void* LZ4HC_Data, const char* source, char* dest, int inputSize, int maxOutputSize, int compressionLevel);
116+
117+
/*
118+
These functions allow the compression of dependent blocks, where each block benefits from prior 64 KB within preceding blocks.
119+
In order to achieve this, it is necessary to start creating the LZ4HC Data Structure, thanks to the function :
120+
121+
void* LZ4_createHC (const char* inputBuffer);
122+
The result of the function is the (void*) pointer on the LZ4HC Data Structure.
123+
This pointer will be needed in all other functions.
124+
If the pointer returned is NULL, then the allocation has failed, and compression must be aborted.
125+
The only parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
126+
The input buffer must be already allocated, and size at least 192KB.
127+
'inputBuffer' will also be the 'const char* source' of the first block.
128+
129+
All blocks are expected to lay next to each other within the input buffer, starting from 'inputBuffer'.
130+
To compress each block, use either LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue().
131+
Their behavior are identical to LZ4_compressHC() or LZ4_compressHC_limitedOutput(),
132+
but require the LZ4HC Data Structure as their first argument, and check that each block starts right after the previous one.
133+
If next block does not begin immediately after the previous one, the compression will fail (return 0).
134+
135+
When it's no longer possible to lay the next block after the previous one (not enough space left into input buffer), a call to :
136+
char* LZ4_slideInputBufferHC(void* LZ4HC_Data);
137+
must be performed. It will typically copy the latest 64KB of input at the beginning of input buffer.
138+
Note that, for this function to work properly, minimum size of an input buffer must be 192KB.
139+
==> The memory position where the next input data block must start is provided as the result of the function.
140+
141+
Compression can then resume, using LZ4_compressHC_continue() or LZ4_compressHC_limitedOutput_continue(), as usual.
142+
143+
When compression is completed, a call to LZ4_freeHC() will release the memory used by the LZ4HC Data Structure.
144+
*/
145+
146+
int LZ4_sizeofStreamStateHC(void);
147+
int LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
148+
149+
/*
150+
These functions achieve the same result as :
151+
void* LZ4_createHC (const char* inputBuffer);
152+
153+
They are provided here to allow the user program to allocate memory using its own routines.
154+
155+
To know how much space must be allocated, use LZ4_sizeofStreamStateHC();
156+
Note also that space must be aligned for pointers (32 or 64 bits).
157+
158+
Once space is allocated, you must initialize it using : LZ4_resetStreamStateHC(void* state, const char* inputBuffer);
159+
void* state is a pointer to the space allocated.
160+
It must be aligned for pointers (32 or 64 bits), and be large enough.
161+
The parameter 'const char* inputBuffer' must, obviously, point at the beginning of input buffer.
162+
The input buffer must be already allocated, and size at least 192KB.
163+
'inputBuffer' will also be the 'const char* source' of the first block.
164+
165+
The same space can be re-used multiple times, just by initializing it each time with LZ4_resetStreamState().
166+
return value of LZ4_resetStreamStateHC() must be 0 is OK.
167+
Any other value means there was an error (typically, state is not aligned for pointers (32 or 64 bits)).
168+
*/
169+
170+
171+
#if defined (__cplusplus)
172+
}
173+
#endif

0 commit comments

Comments
 (0)