Merge pull request #77 from SpringMT/feature/dictionary-standardized

Feature/dictionary standardized

Author: SpringMT

README.md

@@ -34,20 +34,22 @@ Or install it yourself as:
 require 'zstd-ruby'
 ```
 
-### Simple Compression
+### Compression
+
+#### Simple Compression
 
 ```ruby
 compressed_data = Zstd.compress(data)
-compressed_data = Zstd.compress(data, complession_level) # default compression_level is 0
+compressed_data = Zstd.compress(data, level: complession_level) # default compression_level is 3
 ```
 
-### Compression using Dictionary
+#### Compression with Dictionary
 ```ruby
 # dictionary is supposed to have been created using `zstd --train`
-compressed_using_dict = Zstd.compress_using_dict("", IO.read('dictionary_file'))
+compressed_using_dict = Zstd.compress("", dict: IO.read('dictionary_file'))
 ```
 
-### Streaming Compression
+#### Streaming Compression
 ```ruby
 stream = Zstd::StreamingCompress.new
 stream << "abc" << "def"
@@ -66,7 +68,7 @@ res << stream.compress("def")
 res << stream.finish
 ```
 
-### Streaming Compression using Dictionary
+#### Streaming Compression with Dictionary
 ```ruby
 stream = Zstd::StreamingCompress.new(dict: IO.read('dictionary_file'))
 stream << "abc" << "def"
@@ -75,19 +77,30 @@ stream << "ghi"
 res << stream.finish
 ```
 
-### Simple Decompression
+#### Streaming Compression with level and Dictionary
+```ruby
+stream = Zstd::StreamingCompress.new(level: 5, dict: IO.read('dictionary_file'))
+stream << "abc" << "def"
+res = stream.flush
+stream << "ghi"
+res << stream.finish
+```
+
+### Decompression
+
+#### Simple Decompression
 
 ```ruby
 data = Zstd.decompress(compressed_data)
 ```
 
-### Decomporession using Dictionary
+#### Decompression with Dictionary
 ```ruby
 # dictionary is supposed to have been created using `zstd --train`
-Zstd.decompress_using_dict(compressed_using_dict, IO.read('dictionary_file'))
+Zstd.decompress(compressed_using_dict, dict: IO.read('dictionary_file'))
 ```
 
-### Streaming Decompression
+#### Streaming Decompression
 ```ruby
 cstr = "" # Compressed data
 stream = Zstd::StreamingDecompress.new
@@ -96,7 +109,7 @@ result << stream.decompress(cstr[0, 10])
 result << stream.decompress(cstr[10..-1])
 ```
 
-### Streaming Decompression using dictionary
+#### Streaming Decompression with dictionary
 ```ruby
 cstr = "" # Compressed data
 stream = Zstd::StreamingDecompress.new(dict: IO.read('dictionary_file'))

benchmarks/results/city.json.gzip

@@ -14,11 +14,11 @@
 json_string = json_data.to_json
 
 i = 0
-
+start_time = Time.now
 while true do
   Zstd.compress(json_string)
   if ((i % 1000) == 0 )
-    puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+    puts "sec:#{Time.now - start_time}\tcount:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
   end
   i += 1
 end

benchmarks/zstd_compress_memory.rb

@@ -16,7 +16,7 @@
 while true do
   Zstd.decompress IO.read("./results/#{sample_file_name}.zstd")
   if ((i % 1000) == 0 )
-     puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+     puts "sec:#{Time.now - start_time}\tcount:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
   end
   i += 1
 end

benchmarks/zstd_decompress_memory.rb

@@ -13,6 +13,7 @@
 json_string = IO.read("./samples/#{sample_file_name}")
 
 i = 0
+start_time = Time.now
 while true do
   stream = Zstd::StreamingCompress.new
   stream << json_string[0, 5]
@@ -21,7 +22,7 @@
   res << stream.finish
   if ((i % 1000) == 0 )
     GC.start
-    puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+    puts "sec:#{Time.now - start_time}\tcount:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
   end
   i += 1
 end

benchmarks/zstd_streaming_compress_memory.rb

@@ -12,6 +12,7 @@
 
 cstr = IO.read("./results/#{sample_file_name}.zstd")
 i = 0
+start_time = Time.now
 while true do
   stream = Zstd::StreamingDecompress.new
   result = ''
@@ -20,7 +21,7 @@
 
   if ((i % 1000) == 0 )
     GC.start
-    puts "count:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
+    puts "sec:#{Time.now - start_time}\tcount:#{i}\truby_memory:#{ObjectSpace.memsize_of_all/1000}\tobject_count:#{ObjectSpace.count_objects}\trss:#{`ps -o rss= -p #{Process.pid}`.to_i}"
   end
   i += 1
 end

benchmarks/zstd_streaming_decompress_memory.rb

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "sinatra"
+gem "rackup"
+gem "zstd-ruby", path: "../../"

examples/sinatra/Gemfile

@@ -0,0 +1,41 @@
+PATH
+  remote: ../..
+  specs:
+    zstd-ruby (1.5.6.1)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    base64 (0.2.0)
+    mustermann (3.0.0)
+      ruby2_keywords (~> 0.0.1)
+    rack (3.0.10)
+    rack-protection (4.0.0)
+      base64 (>= 0.1.0)
+      rack (>= 3.0.0, < 4)
+    rack-session (2.0.0)
+      rack (>= 3.0.0)
+    rackup (2.1.0)
+      rack (>= 3)
+      webrick (~> 1.8)
+    ruby2_keywords (0.0.5)
+    sinatra (4.0.0)
+      mustermann (~> 3.0)
+      rack (>= 3.0.0, < 4)
+      rack-protection (= 4.0.0)
+      rack-session (>= 2.0.0, < 3)
+      tilt (~> 2.0)
+    tilt (2.3.0)
+    webrick (1.8.1)
+
+PLATFORMS
+  arm64-darwin-21
+  ruby
+
+DEPENDENCIES
+  rackup
+  sinatra
+  zstd-ruby!
+
+BUNDLED WITH
+   2.5.7

examples/sinatra/Gemfile.lock

@@ -0,0 +1,7 @@
+require 'sinatra'
+require 'zstd-ruby'
+
+get '/' do
+  headers["Content-Encoding"] = "zstd"
+  Zstd.compress('Hello world!')
+end

examples/sinatra/app.rb

@@ -1,7 +1,7 @@
 #ifndef ZSTD_RUBY_H
 #define ZSTD_RUBY_H 1
 
-#include "ruby.h"
+#include <ruby.h>
 #include "./libzstd/zstd.h"
 
 static int convert_compression_level(VALUE compression_level_value)
@@ -12,4 +12,55 @@ static int convert_compression_level(VALUE compression_level_value)
   return NUM2INT(compression_level_value);
 }
 
+static size_t zstd_compress(ZSTD_CCtx* const ctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective endOp)
+{
+  return ZSTD_compressStream2(ctx, output, input, endOp);
+}
+
+static void set_compress_params(ZSTD_CCtx* const ctx, VALUE level_from_args, VALUE kwargs)
+{
+  ID kwargs_keys[2];
+  kwargs_keys[0] = rb_intern("level");
+  kwargs_keys[1] = rb_intern("dict");
+  VALUE kwargs_values[2];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 2, kwargs_values);
+
+  int compression_level = ZSTD_CLEVEL_DEFAULT;
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    compression_level = convert_compression_level(kwargs_values[0]);
+  } else if (!NIL_P(level_from_args)) {
+    rb_warn("`level` in args is deprecated; use keyword args `level:` instead.");
+    compression_level = convert_compression_level(level_from_args);
+  }
+  ZSTD_CCtx_setParameter(ctx, ZSTD_c_compressionLevel, compression_level);
+
+  if (kwargs_values[1] != Qundef && kwargs_values[1] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[1]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[1]);
+    size_t load_dict_ret = ZSTD_CCtx_loadDictionary(ctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeCCtx(ctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
+static void set_decompress_params(ZSTD_DCtx* const dctx, VALUE kwargs)
+{
+  ID kwargs_keys[1];
+  kwargs_keys[0] = rb_intern("dict");
+  VALUE kwargs_values[1];
+  rb_get_kwargs(kwargs, kwargs_keys, 0, 1, kwargs_values);
+
+  if (kwargs_values[0] != Qundef && kwargs_values[0] != Qnil) {
+    char* dict_buffer = RSTRING_PTR(kwargs_values[0]);
+    size_t dict_size = RSTRING_LEN(kwargs_values[0]);
+    size_t load_dict_ret = ZSTD_DCtx_loadDictionary(dctx, dict_buffer, dict_size);
+    if (ZSTD_isError(load_dict_ret)) {
+      ZSTD_freeDCtx(dctx);
+      rb_raise(rb_eRuntimeError, "%s", "ZSTD_CCtx_loadDictionary failed");
+    }
+  }
+}
+
 #endif /* ZSTD_RUBY_H */